From 2a531ee06f95a26612e480edb81804dd4f6103b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Fri, 10 Jul 2026 13:33:15 +0800 Subject: [PATCH 1/5] refactor(axvm): consolidate architecture-specific code --- os/axvisor/src/config.rs | 6 +- .../src/{ => arch/aarch64}/boot/fdt/create.rs | 0 .../src/{ => arch/aarch64}/boot/fdt/device.rs | 0 .../fdt/loongarch64/guest_firmware_dtb.rs | 0 .../aarch64}/boot/fdt/loongarch64/mod.rs | 0 .../aarch64}/boot/fdt/loongarch64/property.rs | 0 .../src/{ => arch/aarch64}/boot/fdt/mod.rs | 0 .../src/{ => arch/aarch64}/boot/fdt/parser.rs | 0 .../src/{ => arch/aarch64}/boot/fdt/print.rs | 0 .../src/{ => arch/aarch64}/boot/fdt/tree.rs | 0 .../{ => arch/aarch64}/boot/fdt/tree_tests.rs | 0 .../axvm/src/{host => arch/aarch64}/gic.rs | 0 virtualization/axvm/src/arch/aarch64/mod.rs | 3 +- .../boot/guest_platform/loongarch64/mod.rs | 0 .../boot/guest_platform/loongarch64/probe.rs | 0 .../guest_platform/loongarch64/resources.rs | 0 .../loongarch64}/boot/guest_platform/mod.rs | 0 .../loongarch64/irq.rs} | 0 .../axvm/src/arch/loongarch64/mod.rs | 3 +- virtualization/axvm/src/arch/mod.rs | 212 +++++++++- .../src/{irq/riscv.rs => arch/riscv64/irq.rs} | 2 +- virtualization/axvm/src/arch/riscv64/mod.rs | 5 +- .../{ => arch/x86_64}/boot/images/linux.rs | 0 .../src/{ => arch/x86_64}/boot/images/mod.rs | 0 .../x86_64}/boot/images/x86/boot_params.rs | 0 .../x86_64}/boot/images/x86/linux.rs | 0 .../x86_64}/boot/images/x86/linux_boot.rs | 0 .../{ => arch/x86_64}/boot/images/x86/mod.rs | 0 .../x86_64}/boot/images/x86/mptable.rs | 0 .../x86_64}/boot/images/x86/multiboot.rs | 0 virtualization/axvm/src/arch/x86_64/host.rs | 385 +++++++++++++++++ .../x86_irq.rs => arch/x86_64/irq.rs} | 77 ++-- virtualization/axvm/src/arch/x86_64/mod.rs | 18 +- .../{host/x86_port.rs => arch/x86_64/port.rs} | 0 virtualization/axvm/src/boot/mod.rs | 25 +- virtualization/axvm/src/host/arceos.rs | 392 +----------------- virtualization/axvm/src/host/mod.rs | 9 +- virtualization/axvm/src/host/traits.rs | 14 - virtualization/axvm/src/irq/mod.rs | 14 +- virtualization/axvm/src/lib.rs | 59 +-- virtualization/axvm/src/manager.rs | 8 +- virtualization/axvm/src/runtime/mod.rs | 34 -- virtualization/axvm/src/runtime/vcpus.rs | 19 +- virtualization/axvm/src/timer.rs | 42 +- virtualization/axvm/src/vcpu.rs | 18 +- virtualization/axvm/src/vm.rs | 7 +- virtualization/axvm/src/vm/prepare.rs | 23 +- .../axvm/src/vm/prepare/address_space.rs | 21 +- virtualization/axvm/src/vm/prepare/devices.rs | 89 +--- .../axvm/tests/arch_boundary_contract.rs | 67 +++ 50 files changed, 765 insertions(+), 787 deletions(-) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/create.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/device.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/loongarch64/guest_firmware_dtb.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/loongarch64/mod.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/loongarch64/property.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/mod.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/parser.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/print.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/tree.rs (100%) rename virtualization/axvm/src/{ => arch/aarch64}/boot/fdt/tree_tests.rs (100%) rename virtualization/axvm/src/{host => arch/aarch64}/gic.rs (100%) rename virtualization/axvm/src/{ => arch/loongarch64}/boot/guest_platform/loongarch64/mod.rs (100%) rename virtualization/axvm/src/{ => arch/loongarch64}/boot/guest_platform/loongarch64/probe.rs (100%) rename virtualization/axvm/src/{ => arch/loongarch64}/boot/guest_platform/loongarch64/resources.rs (100%) rename virtualization/axvm/src/{ => arch/loongarch64}/boot/guest_platform/mod.rs (100%) rename virtualization/axvm/src/{runtime/loongarch_irq.rs => arch/loongarch64/irq.rs} (100%) rename virtualization/axvm/src/{irq/riscv.rs => arch/riscv64/irq.rs} (99%) rename virtualization/axvm/src/{ => arch/x86_64}/boot/images/linux.rs (100%) rename virtualization/axvm/src/{ => arch/x86_64}/boot/images/mod.rs (100%) rename virtualization/axvm/src/{ => arch/x86_64}/boot/images/x86/boot_params.rs (100%) rename virtualization/axvm/src/{ => arch/x86_64}/boot/images/x86/linux.rs (100%) rename virtualization/axvm/src/{ => arch/x86_64}/boot/images/x86/linux_boot.rs (100%) rename virtualization/axvm/src/{ => arch/x86_64}/boot/images/x86/mod.rs (100%) rename virtualization/axvm/src/{ => arch/x86_64}/boot/images/x86/mptable.rs (100%) rename virtualization/axvm/src/{ => arch/x86_64}/boot/images/x86/multiboot.rs (100%) create mode 100644 virtualization/axvm/src/arch/x86_64/host.rs rename virtualization/axvm/src/{runtime/x86_irq.rs => arch/x86_64/irq.rs} (95%) rename virtualization/axvm/src/{host/x86_port.rs => arch/x86_64/port.rs} (100%) diff --git a/os/axvisor/src/config.rs b/os/axvisor/src/config.rs index 2dcc1d7bae..dbd1996e09 100644 --- a/os/axvisor/src/config.rs +++ b/os/axvisor/src/config.rs @@ -35,11 +35,11 @@ use axvm::{ use axvmconfig::{AxVMCrateConfig, VMType}; #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use axvm::boot::handle_fdt_operations; +use axvm::boot::fdt::handle_fdt_operations; +#[cfg(target_arch = "loongarch64")] +use axvm::boot::fdt::{handle_fdt_operations, init_guest_boot_resources}; #[cfg(target_arch = "x86_64")] use axvm::boot::is_x86_linux_image_config; -#[cfg(target_arch = "loongarch64")] -use axvm::boot::{handle_fdt_operations, init_guest_boot_resources}; /// Default BIOS load GPA for x86_64 built-in BIOS. #[cfg(target_arch = "x86_64")] diff --git a/virtualization/axvm/src/boot/fdt/create.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/create.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs diff --git a/virtualization/axvm/src/boot/fdt/device.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/device.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/device.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/device.rs diff --git a/virtualization/axvm/src/boot/fdt/loongarch64/guest_firmware_dtb.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/guest_firmware_dtb.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/loongarch64/guest_firmware_dtb.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/guest_firmware_dtb.rs diff --git a/virtualization/axvm/src/boot/fdt/loongarch64/mod.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/mod.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/loongarch64/mod.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/mod.rs diff --git a/virtualization/axvm/src/boot/fdt/loongarch64/property.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/property.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/loongarch64/property.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/property.rs diff --git a/virtualization/axvm/src/boot/fdt/mod.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/mod.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs diff --git a/virtualization/axvm/src/boot/fdt/parser.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/parser.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs diff --git a/virtualization/axvm/src/boot/fdt/print.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/print.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/print.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/print.rs diff --git a/virtualization/axvm/src/boot/fdt/tree.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/tree.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/tree.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/tree.rs diff --git a/virtualization/axvm/src/boot/fdt/tree_tests.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/tree_tests.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/tree_tests.rs rename to virtualization/axvm/src/arch/aarch64/boot/fdt/tree_tests.rs diff --git a/virtualization/axvm/src/host/gic.rs b/virtualization/axvm/src/arch/aarch64/gic.rs similarity index 100% rename from virtualization/axvm/src/host/gic.rs rename to virtualization/axvm/src/arch/aarch64/gic.rs diff --git a/virtualization/axvm/src/arch/aarch64/mod.rs b/virtualization/axvm/src/arch/aarch64/mod.rs index a1600e21e9..871ad43b5d 100644 --- a/virtualization/axvm/src/arch/aarch64/mod.rs +++ b/virtualization/axvm/src/arch/aarch64/mod.rs @@ -25,8 +25,9 @@ use super::{ SysRegReadExit, SysRegWriteExit, VcpuCreateContext, VcpuRunAction, VcpuSetupContext, target_phys_cpu_ids, }; -use crate::host::{HostCpu, HostMemory, HostTime, default_host, gic}; +use crate::host::{HostCpu, HostMemory, HostTime, default_host}; +mod gic; mod npt; pub(crate) struct Aarch64Arch; diff --git a/virtualization/axvm/src/boot/guest_platform/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/mod.rs similarity index 100% rename from virtualization/axvm/src/boot/guest_platform/loongarch64/mod.rs rename to virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/mod.rs diff --git a/virtualization/axvm/src/boot/guest_platform/loongarch64/probe.rs b/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/probe.rs similarity index 100% rename from virtualization/axvm/src/boot/guest_platform/loongarch64/probe.rs rename to virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/probe.rs diff --git a/virtualization/axvm/src/boot/guest_platform/loongarch64/resources.rs b/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/resources.rs similarity index 100% rename from virtualization/axvm/src/boot/guest_platform/loongarch64/resources.rs rename to virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/resources.rs diff --git a/virtualization/axvm/src/boot/guest_platform/mod.rs b/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/mod.rs similarity index 100% rename from virtualization/axvm/src/boot/guest_platform/mod.rs rename to virtualization/axvm/src/arch/loongarch64/boot/guest_platform/mod.rs diff --git a/virtualization/axvm/src/runtime/loongarch_irq.rs b/virtualization/axvm/src/arch/loongarch64/irq.rs similarity index 100% rename from virtualization/axvm/src/runtime/loongarch_irq.rs rename to virtualization/axvm/src/arch/loongarch64/irq.rs diff --git a/virtualization/axvm/src/arch/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/mod.rs index bbec7f0bf7..14e804b9b1 100644 --- a/virtualization/axvm/src/arch/loongarch64/mod.rs +++ b/virtualization/axvm/src/arch/loongarch64/mod.rs @@ -21,6 +21,7 @@ use super::{ }; use crate::host::{HostMemory, HostTime, default_host}; +pub(crate) mod irq; mod npt; pub(crate) struct LoongArch64Arch; @@ -85,7 +86,7 @@ impl ArchOps for LoongArch64Arch { } fn register_platform_irq_injector() { - crate::runtime::loongarch_irq::register_platform_irq_injector(); + irq::register_platform_irq_injector(); } fn inject_pending_interrupt( diff --git a/virtualization/axvm/src/arch/mod.rs b/virtualization/axvm/src/arch/mod.rs index eff1db411f..8de28e7f10 100644 --- a/virtualization/axvm/src/arch/mod.rs +++ b/virtualization/axvm/src/arch/mod.rs @@ -20,6 +20,14 @@ use crate::StopReason; #[cfg(target_arch = "aarch64")] mod aarch64; +#[path = "x86_64/host.rs"] +pub mod arceos; +#[path = "aarch64/boot/fdt/mod.rs"] +pub mod fdt; +#[path = "loongarch64/boot/guest_platform/mod.rs"] +pub mod guest_platform; +#[path = "x86_64/boot/images/mod.rs"] +pub mod images; #[cfg(target_arch = "loongarch64")] mod loongarch64; mod npt; @@ -28,6 +36,35 @@ mod riscv64; #[cfg(target_arch = "x86_64")] mod x86_64; +/// Architecture-specific public compatibility exports. +/// +/// The parent crate re-exports this module so legacy AxVM API paths remain +/// stable while target selection stays contained within `arch`. +pub mod platform { + #[cfg(all( + any(target_arch = "x86_64", target_arch = "loongarch64"), + any(feature = "fs", feature = "host-fs") + ))] + pub use super::arceos::shutdown_host_filesystems; + #[cfg(any( + target_arch = "aarch64", + target_arch = "loongarch64", + target_arch = "riscv64" + ))] + pub use super::arceos::{host_fdt_bootarg, phys_to_virt as host_phys_to_virt}; + #[cfg(target_arch = "loongarch64")] + pub use super::loongarch64::irq::{ + register_guest_irq_route as register_loongarch_guest_irq_route, + unregister_guest_irq_routes as unregister_loongarch_guest_irq_routes, + }; + #[cfg(target_arch = "x86_64")] + pub use super::x86_64::irq::{ + register_ioapic_irq_forwarding_activator as register_x86_ioapic_irq_forwarding_activator, + register_ioapic_irq_forwarding_route as register_x86_ioapic_irq_forwarding_route, + register_ioapic_irq_forwarding_route_with_trigger as register_x86_ioapic_irq_forwarding_route_with_trigger, + }; +} + #[cfg(target_arch = "aarch64")] pub(crate) type CurrentArch = aarch64::Aarch64Arch; #[cfg(target_arch = "loongarch64")] @@ -51,19 +88,6 @@ pub(crate) fn new_nested_page_table(levels: usize) -> AxResult AxResult { - x86_64::register_arch_device(config, devices) -} - -#[cfg(all(target_arch = "x86_64", feature = "vmx"))] -pub(crate) fn x86_apic_access_page_addr() -> axvm_types::HostPhysAddr { - x86_64::x86_apic_access_page_addr() -} - pub(crate) fn nested_paging_config( root_paddr: PhysAddr, levels: usize, @@ -72,6 +96,161 @@ pub(crate) fn nested_paging_config( CurrentArch::nested_paging_config(root_paddr, levels, vcpu_mappings) } +/// Creates the VM interrupt fabric and registers architecture-owned factories. +#[cfg(target_arch = "riscv64")] +pub(crate) fn configure_interrupt_fabric( + factories: &mut axdevice::DeviceFactoryRegistry, + mode: VMInterruptMode, + configs: &[axvm_types::EmulatedDeviceConfig], +) -> AxResult { + riscv64::irq::configure(factories, mode, configs) +} + +/// Creates the VM interrupt fabric when no architecture-owned backend is needed. +#[cfg(any( + target_arch = "aarch64", + target_arch = "loongarch64", + target_arch = "x86_64" +))] +pub(crate) fn configure_interrupt_fabric( + _factories: &mut axdevice::DeviceFactoryRegistry, + mode: VMInterruptMode, + _configs: &[axvm_types::EmulatedDeviceConfig], +) -> AxResult { + Ok(crate::InterruptFabric::new(mode)) +} + +/// Adds architecture-owned guest physical ranges to the address-layout planner. +#[cfg(all(target_arch = "x86_64", feature = "vmx"))] +pub(crate) fn append_arch_owned_regions(regions: &mut Vec) { + regions.push(crate::layout::GuestOwnedRegion::new( + x86_vcpu::X86_APIC_ACCESS_GPA, + ax_memory_addr::PAGE_SIZE_4K, + crate::layout::VmRegionKind::Reserved, + )); +} + +/// Adds no additional guest physical ranges for this architecture configuration. +#[cfg(not(all(target_arch = "x86_64", feature = "vmx")))] +pub(crate) fn append_arch_owned_regions(_regions: &mut Vec) {} + +/// Installs architecture-owned stage-2 mappings after the generic layout. +#[cfg(all(target_arch = "x86_64", feature = "vmx"))] +pub(crate) fn map_arch_address_space( + address_space: &mut axaddrspace::AddrSpace, +) -> AxResult { + address_space.map_linear( + axvm_types::GuestPhysAddr::from(x86_vcpu::X86_APIC_ACCESS_GPA), + x86_64::x86_apic_access_page_addr(), + ax_memory_addr::PAGE_SIZE_4K, + axvm_types::MappingFlags::DEVICE + | axvm_types::MappingFlags::READ + | axvm_types::MappingFlags::WRITE, + ) +} + +/// Installs no additional stage-2 mappings for this architecture configuration. +#[cfg(not(all(target_arch = "x86_64", feature = "vmx")))] +pub(crate) fn map_arch_address_space( + _address_space: &mut axaddrspace::AddrSpace, +) -> AxResult { + Ok(()) +} + +/// Registers the host timer callback required by the active architecture. +#[cfg(target_arch = "loongarch64")] +pub(crate) fn register_timer_callback() { + ax_std::os::arceos::modules::ax_task::register_timer_callback(|_| crate::check_timer_events()); +} + +/// Registers no timer callback when the host timer interrupts directly. +#[cfg(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +))] +pub(crate) fn register_timer_callback() {} + +/// Registers devices and passthrough resources owned by the active architecture. +#[cfg(target_arch = "aarch64")] +pub(crate) fn register_arch_devices( + vm: &crate::AxVM, + config: &crate::config::AxVMConfig, + devices: &mut axdevice::AxVmDevices, +) -> AxResult { + use alloc::sync::Arc; + + use ax_errno::ax_err_type; + use axdevice_base::DeviceRegistry as _; + + if config.interrupt_mode() == VMInterruptMode::Passthrough { + let cpu_id = vm.id() - 1; // FIXME: get the real CPU id. + let mut gicd_found = false; + for device in devices.devices() { + if let Some(gicd) = device.as_any().downcast_ref::() { + for spi in config.pass_through_spis() { + gicd.assign_irq(*spi + 32, cpu_id, (0, 0, 0, cpu_id as _)); + } + gicd_found = true; + break; + } + } + if !gicd_found { + warn!("Failed to assign SPIs: No VGicD found in device list"); + } + } else { + for device in axdevice::create_vtimer_devices() { + devices + .register(Arc::from(device) as Arc) + .map_err(|err| ax_err_type!(InvalidInput, format!("register vtimer: {err:?}")))?; + } + } + Ok(()) +} + +/// Registers devices and passthrough resources owned by the active architecture. +#[cfg(target_arch = "x86_64")] +pub(crate) fn register_arch_devices( + _vm: &crate::AxVM, + config: &crate::config::AxVMConfig, + devices: &mut axdevice::AxVmDevices, +) -> AxResult { + use alloc::{format, sync::Arc}; + + use ax_errno::ax_err_type; + use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; + + for port in config.pass_through_ports() { + let passthrough = Arc::new(x86_64::port::HostPortPassthrough::new( + port.base, + port.length, + )?); + let range = passthrough.address_range(); + debug!( + "PT port region: [{:#x}~{:#x}]", + range.start.number(), + range.end.number(), + ); + devices + .register(PortDeviceAdapter::from_arc(passthrough)) + .map_err(|err| ax_err_type!(InvalidInput, format!("register PT port: {err:?}")))?; + } + for config in config.emu_devices() { + x86_64::register_arch_device(config, devices)?; + } + Ok(()) +} + +/// Registers devices and passthrough resources owned by the active architecture. +#[cfg(any(target_arch = "loongarch64", target_arch = "riscv64"))] +pub(crate) fn register_arch_devices( + _vm: &crate::AxVM, + _config: &crate::config::AxVMConfig, + _devices: &mut axdevice::AxVmDevices, +) -> AxResult { + Ok(()) +} + /// Runtime scheduler action selected after an architecture-local vCPU exit. #[derive(Debug)] pub(crate) enum VcpuRunAction { @@ -286,12 +465,15 @@ pub(crate) trait ArchOps { targets } - #[cfg(not(target_arch = "x86_64"))] + #[expect( + dead_code, + reason = "secondary vCPU boot is unsupported on the x86 build" + )] fn set_vcpu_on_args(vcpu: &crate::vm::AxVCpuRef, _vcpu_id: usize, arg: usize) { vcpu.set_gpr(0, arg); } - #[cfg(not(target_arch = "x86_64"))] + #[expect(dead_code, reason = "CPU-up exits are unsupported on the x86 build")] fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { vcpu.set_gpr(0, 0); } diff --git a/virtualization/axvm/src/irq/riscv.rs b/virtualization/axvm/src/arch/riscv64/irq.rs similarity index 99% rename from virtualization/axvm/src/irq/riscv.rs rename to virtualization/axvm/src/arch/riscv64/irq.rs index c514e9dfa8..af994514dc 100644 --- a/virtualization/axvm/src/irq/riscv.rs +++ b/virtualization/axvm/src/arch/riscv64/irq.rs @@ -27,7 +27,7 @@ use riscv_vplic::{ PLIC_CONTEXT_CLAIM_COMPLETE_OFFSET, PLIC_CONTEXT_CTRL_OFFSET, PLIC_CONTEXT_STRIDE, VPlicGlobal, }; -use super::InterruptFabric; +use crate::irq::InterruptFabric; struct RiscvPlicIrqSink { vplic: Arc, diff --git a/virtualization/axvm/src/arch/riscv64/mod.rs b/virtualization/axvm/src/arch/riscv64/mod.rs index a50f873445..72226a0658 100644 --- a/virtualization/axvm/src/arch/riscv64/mod.rs +++ b/virtualization/axvm/src/arch/riscv64/mod.rs @@ -24,6 +24,7 @@ use crate::{ host::{HostMemory, default_host}, }; +mod irq; mod npt; pub(crate) struct Riscv64Arch; @@ -113,7 +114,7 @@ impl ArchOps for Riscv64Arch { return; }; let irq_sources = vm.with_config(|config| config.pass_through_irqs().to_vec()); - crate::irq::set_riscv_virtual_irq_targets(cpu_id, &irq_sources); + irq::set_virtual_irq_targets(cpu_id, &irq_sources); } fn vcpu_affinities( @@ -459,7 +460,7 @@ impl RiscvVplicHostIf for RiscvVplicHostIfImpl { } fn register_platform_irq_injector() { - crate::irq::register_riscv_virtual_irq_injector(inject_virtual_irq); + irq::register_virtual_irq_injector(inject_virtual_irq); } fn first_cpu_in_mask(mask: usize) -> Option { diff --git a/virtualization/axvm/src/boot/images/linux.rs b/virtualization/axvm/src/arch/x86_64/boot/images/linux.rs similarity index 100% rename from virtualization/axvm/src/boot/images/linux.rs rename to virtualization/axvm/src/arch/x86_64/boot/images/linux.rs diff --git a/virtualization/axvm/src/boot/images/mod.rs b/virtualization/axvm/src/arch/x86_64/boot/images/mod.rs similarity index 100% rename from virtualization/axvm/src/boot/images/mod.rs rename to virtualization/axvm/src/arch/x86_64/boot/images/mod.rs diff --git a/virtualization/axvm/src/boot/images/x86/boot_params.rs b/virtualization/axvm/src/arch/x86_64/boot/images/x86/boot_params.rs similarity index 100% rename from virtualization/axvm/src/boot/images/x86/boot_params.rs rename to virtualization/axvm/src/arch/x86_64/boot/images/x86/boot_params.rs diff --git a/virtualization/axvm/src/boot/images/x86/linux.rs b/virtualization/axvm/src/arch/x86_64/boot/images/x86/linux.rs similarity index 100% rename from virtualization/axvm/src/boot/images/x86/linux.rs rename to virtualization/axvm/src/arch/x86_64/boot/images/x86/linux.rs diff --git a/virtualization/axvm/src/boot/images/x86/linux_boot.rs b/virtualization/axvm/src/arch/x86_64/boot/images/x86/linux_boot.rs similarity index 100% rename from virtualization/axvm/src/boot/images/x86/linux_boot.rs rename to virtualization/axvm/src/arch/x86_64/boot/images/x86/linux_boot.rs diff --git a/virtualization/axvm/src/boot/images/x86/mod.rs b/virtualization/axvm/src/arch/x86_64/boot/images/x86/mod.rs similarity index 100% rename from virtualization/axvm/src/boot/images/x86/mod.rs rename to virtualization/axvm/src/arch/x86_64/boot/images/x86/mod.rs diff --git a/virtualization/axvm/src/boot/images/x86/mptable.rs b/virtualization/axvm/src/arch/x86_64/boot/images/x86/mptable.rs similarity index 100% rename from virtualization/axvm/src/boot/images/x86/mptable.rs rename to virtualization/axvm/src/arch/x86_64/boot/images/x86/mptable.rs diff --git a/virtualization/axvm/src/boot/images/x86/multiboot.rs b/virtualization/axvm/src/arch/x86_64/boot/images/x86/multiboot.rs similarity index 100% rename from virtualization/axvm/src/boot/images/x86/multiboot.rs rename to virtualization/axvm/src/arch/x86_64/boot/images/x86/multiboot.rs diff --git a/virtualization/axvm/src/arch/x86_64/host.rs b/virtualization/axvm/src/arch/x86_64/host.rs new file mode 100644 index 0000000000..99ea4425b4 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/host.rs @@ -0,0 +1,385 @@ +//! Default private ArceOS host adapter for AxVM. + +extern crate alloc; + +use alloc::boxed::Box; +use core::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; + +use ax_errno::AxResult; +use ax_memory_addr::PAGE_SIZE_4K; +use ax_std::{ + os::arceos::{api, modules}, + thread, +}; +use axvm_types::{HostPhysAddr, HostVirtAddr}; + +use crate::{ + arch::{ArchOps, CurrentArch}, + host::{HostConsole, HostCpu, HostMemory, HostPlatform, HostTime}, +}; + +/// Private default host adapter used by [`crate::AxvmRuntime`]. +pub(crate) struct ArceOsHost; + +const CPU_ENABLE_WAIT_TIMEOUT: Duration = Duration::from_secs(5); + +static ARCEOS_HOST: ArceOsHost = ArceOsHost; + +pub(crate) fn arceos_host() -> &'static ArceOsHost { + &ARCEOS_HOST +} + +impl HostMemory for ArceOsHost { + fn alloc_frame(&self) -> Option { + modules::ax_alloc::global_allocator() + .alloc_pages(1, PAGE_SIZE_4K, modules::ax_alloc::UsageKind::PageTable) + .map(|vaddr| self.virt_to_phys(vaddr.into())) + .ok() + } + + fn dealloc_frame(&self, paddr: HostPhysAddr) { + modules::ax_alloc::global_allocator().dealloc_pages( + self.phys_to_virt(paddr).as_usize(), + 1, + modules::ax_alloc::UsageKind::PageTable, + ); + } + + fn alloc_contiguous_frames( + &self, + num_frames: usize, + frame_align: usize, + ) -> Option { + modules::ax_alloc::global_allocator() + .alloc_pages( + num_frames, + frame_align.max(PAGE_SIZE_4K), + modules::ax_alloc::UsageKind::Dma, + ) + .map(|vaddr| self.virt_to_phys(vaddr.into())) + .ok() + } + + fn dealloc_contiguous_frames(&self, paddr: HostPhysAddr, num_frames: usize) { + modules::ax_alloc::global_allocator().dealloc_pages( + self.phys_to_virt(paddr).as_usize(), + num_frames, + modules::ax_alloc::UsageKind::Dma, + ); + } + + fn phys_to_virt(&self, paddr: HostPhysAddr) -> HostVirtAddr { + modules::ax_hal::mem::phys_to_virt(paddr) + } + + fn virt_to_phys(&self, vaddr: HostVirtAddr) -> HostPhysAddr { + modules::ax_hal::mem::virt_to_phys(vaddr) + } +} + +impl HostTime for ArceOsHost { + type CancelToken = usize; + + fn nanos_to_ticks(&self, nanos: u64) -> u64 { + #[cfg(target_arch = "x86_64")] + { + modules::ax_hal::time::nanos_to_ticks(nanos) + } + #[cfg(not(target_arch = "x86_64"))] + { + nanos + } + } + + fn monotonic_time(&self) -> Duration { + modules::ax_hal::time::monotonic_time() + } + + fn set_oneshot_timer(&self, deadline_ns: u64) { + #[cfg(not(target_arch = "loongarch64"))] + modules::ax_hal::time::set_oneshot_timer(deadline_ns); + #[cfg(target_arch = "loongarch64")] + let _ = deadline_ns; + } + + fn register_timer( + &self, + deadline_ns: u64, + callback: Box, + ) -> Self::CancelToken { + crate::timer::register_timer(deadline_ns, callback) + } + + fn cancel_timer(&self, token: Self::CancelToken) { + crate::timer::cancel_timer(token); + } +} + +#[cfg(target_arch = "x86_64")] +pub(crate) fn monotonic_time_nanos() -> u64 { + modules::ax_hal::time::monotonic_time_nanos() +} + +#[cfg(target_arch = "aarch64")] +pub(crate) fn handle_host_irq(vector: usize) -> Option { + modules::ax_hal::irq::handle_irq(vector).then_some(vector) +} + +pub(crate) fn dispatch_host_irq(vector: usize) { + modules::ax_hal::irq::handle_irq(vector); +} + +#[cfg(target_arch = "loongarch64")] +pub(crate) fn set_irq_enabled(raw_irq: usize, enabled: bool) { + let gsi = match u32::try_from(raw_irq) { + Ok(gsi) => gsi, + Err(_) => { + warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: out of GSI range"); + return; + } + }; + let irq = match modules::ax_hal::irq::resolve_irq_source( + modules::ax_hal::irq::IrqSource::AcpiGsi(gsi), + ) { + Ok(irq) => irq, + Err(err) => { + warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: {err:?}"); + return; + } + }; + if let Err(err) = modules::ax_hal::irq::set_enable(irq, enabled) { + warn!( + "failed to set LoongArch passthrough IRQ {raw_irq} ({irq:?}) enabled={enabled}: \ + {err:?}" + ); + } +} + +impl HostCpu for ArceOsHost { + type CpuMask = api::task::AxCpuMask; + + fn cpu_count(&self) -> usize { + modules::ax_hal::cpu_num() + } + + fn this_cpu_id(&self) -> usize { + modules::ax_hal::percpu::this_cpu_id() + } +} + +pub(crate) fn cpu_mask_from_raw_bits(bits: usize) -> api::task::AxCpuMask { + api::task::AxCpuMask::from_raw_bits(bits) +} + +pub(crate) type ArceOsCpuMask = api::task::AxCpuMask; +pub(crate) type ArceOsAxTaskExt = modules::ax_task::AxTaskExt; +pub(crate) type ArceOsAxTaskRef = modules::ax_task::AxTaskRef; +pub(crate) type ArceOsCurrentTask = modules::ax_task::CurrentTask; +pub(crate) type ArceOsTaskInner = modules::ax_task::TaskInner; +pub(crate) type ArceOsWaitQueue = modules::ax_task::WaitQueue; +pub(crate) type ArceOsWaitQueueHandle = api::task::AxWaitQueueHandle; +pub(crate) use modules::ax_task::TaskExt as ArceOsTaskExt; + +pub(crate) fn current_task() -> ArceOsCurrentTask { + modules::ax_task::current() +} + +pub(crate) fn spawn_task(task: ArceOsTaskInner) -> ArceOsAxTaskRef { + modules::ax_task::spawn_task(task) +} + +pub(crate) fn yield_now() { + thread::yield_now(); +} + +pub(crate) fn wait_queue_wait_until( + queue: &api::task::AxWaitQueueHandle, + condition: impl Fn() -> bool, +) { + api::task::ax_wait_queue_wait_until(queue, condition, None); +} + +pub(crate) fn wait_queue_wake(queue: &api::task::AxWaitQueueHandle, count: u32) { + api::task::ax_wait_queue_wake(queue, count); +} + +pub(crate) fn send_ipi(cpu_id: usize) { + if modules::ax_hal::percpu::this_cpu_id() == cpu_id { + return; + } + modules::ax_hal::irq::send_ipi( + modules::ax_hal::irq::ipi_irq(), + modules::ax_hal::irq::IpiTarget::Other { cpu_id }, + ); +} + +fn send_ipi_to_all_except_current(cpu_num: usize) { + if cpu_num <= 1 { + return; + } + let cpu_id = modules::ax_hal::percpu::this_cpu_id(); + modules::ax_hal::irq::send_ipi( + modules::ax_hal::irq::ipi_irq(), + modules::ax_hal::irq::IpiTarget::AllExceptCurrent { cpu_id, cpu_num }, + ); +} + +#[cfg(target_arch = "x86_64")] +pub(crate) type ArceOsIrqContext = modules::ax_hal::irq::IrqContext; +#[cfg(target_arch = "x86_64")] +pub(crate) type ArceOsIrqError = modules::ax_hal::irq::IrqError; +#[cfg(target_arch = "x86_64")] +pub(crate) type ArceOsIrqHandle = modules::ax_hal::irq::IrqHandle; +#[cfg(target_arch = "x86_64")] +pub(crate) type ArceOsIrqId = modules::ax_hal::irq::IrqId; +#[cfg(target_arch = "x86_64")] +pub(crate) type ArceOsIrqReturn = modules::ax_hal::irq::IrqReturn; +#[cfg(target_arch = "x86_64")] +pub(crate) type ArceOsIrqSource = modules::ax_hal::irq::IrqSource; +#[cfg(target_arch = "x86_64")] +pub(crate) fn request_shared_irq( + irq: ArceOsIrqId, + handler: impl FnMut(ArceOsIrqContext) -> ArceOsIrqReturn + Send + 'static, +) -> Result { + modules::ax_hal::irq::request_shared_irq(irq, handler) +} + +#[cfg(target_arch = "x86_64")] +pub(crate) fn make_irq_id(domain: u16, hwirq: u32) -> ArceOsIrqId { + modules::ax_hal::irq::IrqId::new( + modules::ax_hal::irq::IrqDomainId(domain), + modules::ax_hal::irq::HwIrq(hwirq), + ) +} + +#[cfg(all(target_arch = "x86_64", not(test)))] +pub(crate) fn set_irq_enable(irq: ArceOsIrqId, enabled: bool) -> Result<(), ArceOsIrqError> { + modules::ax_hal::irq::set_enable(irq, enabled) +} + +#[cfg(target_arch = "x86_64")] +pub(crate) fn resolve_irq_source(source: ArceOsIrqSource) -> Result { + modules::ax_hal::irq::resolve_irq_source(source) +} + +#[cfg(any( + target_arch = "aarch64", + target_arch = "loongarch64", + target_arch = "riscv64" +))] +pub fn host_fdt_bootarg() -> usize { + modules::ax_hal::dtb::get_bootarg() +} + +#[cfg(any( + target_arch = "aarch64", + target_arch = "loongarch64", + target_arch = "riscv64" +))] +pub fn phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + modules::ax_hal::mem::phys_to_virt(paddr) +} + +#[cfg(all( + any(feature = "fs", feature = "host-fs"), + any(target_arch = "x86_64", target_arch = "loongarch64") +))] +pub fn shutdown_host_filesystems() -> AxResult { + modules::ax_fs_ng::shutdown_filesystems()?; + let released = modules::ax_fs_ng::release_block_irqs_for_passthrough(); + if released != 0 { + info!("Released {released} host filesystem block IRQ registration(s) before passthrough"); + } + Ok(()) +} + +impl HostConsole for ArceOsHost { + fn write_bytes(&self, bytes: &[u8]) { + modules::ax_hal::console::write_bytes(bytes); + } + + fn read_bytes(&self, bytes: &mut [u8]) -> usize { + modules::ax_hal::console::read_bytes(bytes) + } +} + +impl HostPlatform for ArceOsHost { + fn has_hardware_support(&self) -> bool { + CurrentArch::has_hardware_support() + } + + fn enable_virtualization_on_current_cpu(&self) -> AxResult { + crate::timer::init_percpu(); + crate::percpu::init_current_cpu()?; + crate::percpu::enable_current_cpu()?; + crate::percpu::mark_cpu_enabled(self.this_cpu_id()); + Ok(()) + } + + fn enable_virtualization_on_all_cpus(&self) -> AxResult { + static CORES: AtomicUsize = AtomicUsize::new(0); + + info!("Enabling hardware virtualization support on all cores..."); + CORES.store(0, Ordering::Release); + crate::percpu::reset_enabled_cpu_mask(); + + let cpu_count = self.cpu_count(); + let current_cpu = self.this_cpu_id(); + info!("Core {current_cpu} is initializing hardware virtualization support..."); + self.enable_virtualization_on_current_cpu()?; + info!("Hardware virtualization support enabled on core {current_cpu}"); + CORES.store(1, Ordering::Release); + + for cpu_id in 0..cpu_count { + if cpu_id == current_cpu { + continue; + } + let task = modules::ax_task::TaskInner::new( + move || { + let host = arceos_host(); + info!("Core {cpu_id} is initializing hardware virtualization support..."); + host.enable_virtualization_on_current_cpu() + .expect("failed to enable hardware virtualization"); + info!("Hardware virtualization support enabled on core {cpu_id}"); + let _ = CORES.fetch_add(1, Ordering::Release); + }, + alloc::format!("axvm-hv-init-{cpu_id}"), + modules::ax_task::default_task_stack_size(), + ); + task.set_cpumask(::CpuMask::one_shot(cpu_id)); + modules::ax_task::spawn_task(task); + if cpu_id != self.this_cpu_id() { + send_ipi(cpu_id); + } + } + + info!("Waiting for all cores to enable hardware virtualization..."); + let start = self.monotonic_time(); + let mut wait_rounds = 0usize; + while CORES.load(Ordering::Acquire) != cpu_count { + thread::yield_now(); + wait_rounds = wait_rounds.wrapping_add(1); + if wait_rounds.is_multiple_of(256) { + send_ipi_to_all_except_current(cpu_count); + } + if self.monotonic_time().saturating_sub(start) >= CPU_ENABLE_WAIT_TIMEOUT { + break; + } + } + CurrentArch::register_platform_irq_injector(); + let enabled_count = CORES.load(Ordering::Acquire); + if enabled_count == cpu_count { + info!("All cores have enabled hardware virtualization support."); + } else { + warn!( + "Only {enabled_count}/{cpu_count} cores enabled hardware virtualization before \ + timeout; continuing with host CPU mask {:#x}", + crate::percpu::enabled_cpu_mask() + ); + } + Ok(()) + } +} diff --git a/virtualization/axvm/src/runtime/x86_irq.rs b/virtualization/axvm/src/arch/x86_64/irq.rs similarity index 95% rename from virtualization/axvm/src/runtime/x86_irq.rs rename to virtualization/axvm/src/arch/x86_64/irq.rs index 1f5013ee2e..ce802b629c 100644 --- a/virtualization/axvm/src/runtime/x86_irq.rs +++ b/virtualization/axvm/src/arch/x86_64/irq.rs @@ -1,6 +1,7 @@ use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use ax_kspin::SpinRaw as Mutex; +use axvm_types::VmArchVcpuOps; use crate::{ InterruptTriggerMode, @@ -106,15 +107,16 @@ pub fn inject_due_pit_irq0(vm: &VMRef, vcpu: &VCpuRef) { return; }; - vcpu.inject_interrupt_with_trigger( - irq.vector as _, - if irq.level_triggered { - InterruptTriggerMode::LevelTriggered - } else { - InterruptTriggerMode::EdgeTriggered - }, - ) - .unwrap(); + vcpu.get_arch_vcpu() + .inject_interrupt_with_trigger( + irq.vector as _, + if irq.level_triggered { + InterruptTriggerMode::LevelTriggered + } else { + InterruptTriggerMode::EdgeTriggered + }, + ) + .unwrap(); } pub fn inject_pending_serial_irq(vm: &VMRef, vcpu: &VCpuRef) { @@ -135,15 +137,16 @@ pub fn inject_pending_serial_irq(vm: &VMRef, vcpu: &VCpuRef) { }; trace!("Injecting x86 COM1 RX IRQ vector {:#x}", irq.vector); - vcpu.inject_interrupt_with_trigger( - irq.vector as _, - if irq.level_triggered { - InterruptTriggerMode::LevelTriggered - } else { - InterruptTriggerMode::EdgeTriggered - }, - ) - .unwrap(); + vcpu.get_arch_vcpu() + .inject_interrupt_with_trigger( + irq.vector as _, + if irq.level_triggered { + InterruptTriggerMode::LevelTriggered + } else { + InterruptTriggerMode::EdgeTriggered + }, + ) + .unwrap(); } pub fn inject_pending_ioapic_irq_after_eoi(vm: &VMRef, vcpu: &VCpuRef, vector: u8) { @@ -170,15 +173,16 @@ pub fn inject_pending_ioapic_irq_after_eoi(vm: &VMRef, vcpu: &VCpuRef, vector: u "Injecting pending x86 IOAPIC level IRQ vector {:#x} after EOI {vector:#x}", irq.vector ); - vcpu.inject_interrupt_with_trigger( - irq.vector as _, - if irq.level_triggered { - InterruptTriggerMode::LevelTriggered - } else { - InterruptTriggerMode::EdgeTriggered - }, - ) - .unwrap(); + vcpu.get_arch_vcpu() + .inject_interrupt_with_trigger( + irq.vector as _, + if irq.level_triggered { + InterruptTriggerMode::LevelTriggered + } else { + InterruptTriggerMode::EdgeTriggered + }, + ) + .unwrap(); } fn should_rearm_forwarded_host_gsi_after_eoi(pending: Option) -> bool { @@ -436,15 +440,16 @@ fn forward_passthrough_gsi( return false; }; - vcpu.inject_interrupt_with_trigger( - guest_irq.vector as _, - if guest_irq.level_triggered { - InterruptTriggerMode::LevelTriggered - } else { - InterruptTriggerMode::EdgeTriggered - }, - ) - .unwrap(); + vcpu.get_arch_vcpu() + .inject_interrupt_with_trigger( + guest_irq.vector as _, + if guest_irq.level_triggered { + InterruptTriggerMode::LevelTriggered + } else { + InterruptTriggerMode::EdgeTriggered + }, + ) + .unwrap(); true } diff --git a/virtualization/axvm/src/arch/x86_64/mod.rs b/virtualization/axvm/src/arch/x86_64/mod.rs index 5318398bf0..6f6c30a8e3 100644 --- a/virtualization/axvm/src/arch/x86_64/mod.rs +++ b/virtualization/axvm/src/arch/x86_64/mod.rs @@ -34,7 +34,9 @@ use crate::{ vcpu::get_current_vcpu, }; +pub(crate) mod irq; mod npt; +pub(crate) mod port; const QEMU_EXIT_PORT: u16 = 0x604; const QEMU_EXIT_MAGIC: u64 = 0x2000; @@ -95,12 +97,12 @@ impl ArchOps for X86_64Arch { } fn before_first_run(vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { - crate::runtime::x86_irq::enable_ioapic_irq_forwarding(vm, vcpu); + irq::enable_ioapic_irq_forwarding(vm, vcpu); } fn before_vcpu_run(vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { - crate::runtime::x86_irq::drain_pending_ioapic_irqs(vm, vcpu); - crate::runtime::x86_irq::activate_ready_ioapic_forwarding_routes(vm); + irq::drain_pending_ioapic_irqs(vm, vcpu); + irq::activate_ready_ioapic_forwarding_routes(vm); } fn after_external_interrupt( @@ -110,13 +112,13 @@ impl ArchOps for X86_64Arch { ) { crate::host::arceos::dispatch_host_irq(vector); crate::check_timer_events(); - crate::runtime::x86_irq::inject_pending_serial_irq(vm, vcpu); + irq::inject_pending_serial_irq(vm, vcpu); } fn after_preemption_timer(vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { crate::timer::check_events(); - crate::runtime::x86_irq::inject_due_pit_irq0(vm, vcpu); - crate::runtime::x86_irq::inject_pending_serial_irq(vm, vcpu); + irq::inject_due_pit_irq0(vm, vcpu); + irq::inject_pending_serial_irq(vm, vcpu); } fn after_interrupt_end( @@ -125,7 +127,7 @@ impl ArchOps for X86_64Arch { vector: Option, ) { if let Some(vector) = vector { - crate::runtime::x86_irq::inject_pending_ioapic_irq_after_eoi(vm, vcpu, vector); + irq::inject_pending_ioapic_irq_after_eoi(vm, vcpu, vector); } } @@ -134,7 +136,7 @@ impl ArchOps for X86_64Arch { } fn on_last_vcpu_exit(vm_id: usize) { - crate::runtime::x86_irq::disable_ioapic_irq_forwarding_for_vm(vm_id); + irq::disable_ioapic_irq_forwarding_for_vm(vm_id); } fn handle_vcpu_exit_bound( diff --git a/virtualization/axvm/src/host/x86_port.rs b/virtualization/axvm/src/arch/x86_64/port.rs similarity index 100% rename from virtualization/axvm/src/host/x86_port.rs rename to virtualization/axvm/src/arch/x86_64/port.rs diff --git a/virtualization/axvm/src/boot/mod.rs b/virtualization/axvm/src/boot/mod.rs index 0a8c52217f..146404050e 100644 --- a/virtualization/axvm/src/boot/mod.rs +++ b/virtualization/axvm/src/boot/mod.rs @@ -3,30 +3,12 @@ #[cfg(any(feature = "fs", feature = "host-fs"))] use ax_errno::ax_err_type; -pub mod images; mod policy; - -#[cfg(any( - test, - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub mod fdt; -#[cfg(target_arch = "loongarch64")] -pub mod guest_platform; - -#[cfg(target_arch = "loongarch64")] -pub use fdt::handle_fdt_operations; -#[cfg(target_arch = "loongarch64")] -pub use fdt::init_guest_boot_resources; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -pub use fdt::{GuestDtbImage, handle_fdt_operations}; -pub use images::{ImageLoader, get_image_header}; -#[cfg(target_arch = "x86_64")] -pub use images::{is_x86_linux_image_config, x86_qemu_passthrough_block_intx}; +pub use images::*; pub use policy::{GuestAcpiTables, GuestBootDescription, GuestDeviceTree, GuestFdtBuilder}; +pub use crate::arch::{fdt, guest_platform, images}; + /// Build-time image bytes supplied by the hypervisor application. #[derive(Clone, Copy, Debug)] pub struct StaticVmImage { @@ -44,7 +26,6 @@ pub struct StaticVmImage { pub trait BootImageProvider { fn static_vm_images(&self) -> &'static [StaticVmImage]; - #[cfg(target_arch = "loongarch64")] fn static_firmware_images(&self) -> &'static [StaticVmImage] { &[] } diff --git a/virtualization/axvm/src/host/arceos.rs b/virtualization/axvm/src/host/arceos.rs index 15e88fda1c..1ebd60ffb6 100644 --- a/virtualization/axvm/src/host/arceos.rs +++ b/virtualization/axvm/src/host/arceos.rs @@ -1,391 +1,3 @@ -//! Default private ArceOS host adapter for AxVM. +//! Re-export the architecture-owned ArceOS host adapter for common runtime code. -extern crate alloc; - -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] -use alloc::boxed::Box; -use core::{ - sync::atomic::{AtomicUsize, Ordering}, - time::Duration, -}; - -use ax_errno::AxResult; -use ax_memory_addr::PAGE_SIZE_4K; -use ax_std::{ - os::arceos::{api, modules}, - thread, -}; -use axvm_types::{HostPhysAddr, HostVirtAddr}; - -#[cfg(target_arch = "x86_64")] -use crate::host::HostConsole; -use crate::{ - arch::{ArchOps, CurrentArch}, - host::{HostCpu, HostMemory, HostPlatform, HostTime}, -}; - -/// Private default host adapter used by [`crate::AxvmRuntime`]. -pub(crate) struct ArceOsHost; - -const CPU_ENABLE_WAIT_TIMEOUT: Duration = Duration::from_secs(5); - -static ARCEOS_HOST: ArceOsHost = ArceOsHost; - -pub(crate) fn arceos_host() -> &'static ArceOsHost { - &ARCEOS_HOST -} - -impl HostMemory for ArceOsHost { - fn alloc_frame(&self) -> Option { - modules::ax_alloc::global_allocator() - .alloc_pages(1, PAGE_SIZE_4K, modules::ax_alloc::UsageKind::PageTable) - .map(|vaddr| self.virt_to_phys(vaddr.into())) - .ok() - } - - fn dealloc_frame(&self, paddr: HostPhysAddr) { - modules::ax_alloc::global_allocator().dealloc_pages( - self.phys_to_virt(paddr).as_usize(), - 1, - modules::ax_alloc::UsageKind::PageTable, - ); - } - - fn alloc_contiguous_frames( - &self, - num_frames: usize, - frame_align: usize, - ) -> Option { - modules::ax_alloc::global_allocator() - .alloc_pages( - num_frames, - frame_align.max(PAGE_SIZE_4K), - modules::ax_alloc::UsageKind::Dma, - ) - .map(|vaddr| self.virt_to_phys(vaddr.into())) - .ok() - } - - fn dealloc_contiguous_frames(&self, paddr: HostPhysAddr, num_frames: usize) { - modules::ax_alloc::global_allocator().dealloc_pages( - self.phys_to_virt(paddr).as_usize(), - num_frames, - modules::ax_alloc::UsageKind::Dma, - ); - } - - fn phys_to_virt(&self, paddr: HostPhysAddr) -> HostVirtAddr { - modules::ax_hal::mem::phys_to_virt(paddr) - } - - fn virt_to_phys(&self, vaddr: HostVirtAddr) -> HostPhysAddr { - modules::ax_hal::mem::virt_to_phys(vaddr) - } -} - -impl HostTime for ArceOsHost { - type CancelToken = usize; - - #[cfg(target_arch = "x86_64")] - fn nanos_to_ticks(&self, nanos: u64) -> u64 { - modules::ax_hal::time::nanos_to_ticks(nanos) - } - - fn monotonic_time(&self) -> Duration { - modules::ax_hal::time::monotonic_time() - } - - #[cfg(not(target_arch = "loongarch64"))] - fn set_oneshot_timer(&self, deadline_ns: u64) { - modules::ax_hal::time::set_oneshot_timer(deadline_ns); - } - - #[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" - ))] - fn register_timer( - &self, - deadline_ns: u64, - callback: Box, - ) -> Self::CancelToken { - crate::timer::register_timer(deadline_ns, callback) - } - - #[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] - fn cancel_timer(&self, token: Self::CancelToken) { - crate::timer::cancel_timer(token); - } -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn monotonic_time_nanos() -> u64 { - modules::ax_hal::time::monotonic_time_nanos() -} - -#[cfg(target_arch = "aarch64")] -pub(crate) fn handle_host_irq(vector: usize) -> Option { - modules::ax_hal::irq::handle_irq(vector).then_some(vector) -} - -pub(crate) fn dispatch_host_irq(vector: usize) { - modules::ax_hal::irq::handle_irq(vector); -} - -#[cfg(target_arch = "loongarch64")] -pub(crate) fn set_irq_enabled(raw_irq: usize, enabled: bool) { - let gsi = match u32::try_from(raw_irq) { - Ok(gsi) => gsi, - Err(_) => { - warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: out of GSI range"); - return; - } - }; - let irq = match modules::ax_hal::irq::resolve_irq_source( - modules::ax_hal::irq::IrqSource::AcpiGsi(gsi), - ) { - Ok(irq) => irq, - Err(err) => { - warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: {err:?}"); - return; - } - }; - if let Err(err) = modules::ax_hal::irq::set_enable(irq, enabled) { - warn!( - "failed to set LoongArch passthrough IRQ {raw_irq} ({irq:?}) enabled={enabled}: \ - {err:?}" - ); - } -} - -impl HostCpu for ArceOsHost { - type CpuMask = api::task::AxCpuMask; - - fn cpu_count(&self) -> usize { - modules::ax_hal::cpu_num() - } - - fn this_cpu_id(&self) -> usize { - modules::ax_hal::percpu::this_cpu_id() - } -} - -pub(crate) fn cpu_mask_from_raw_bits(bits: usize) -> api::task::AxCpuMask { - api::task::AxCpuMask::from_raw_bits(bits) -} - -pub(crate) type ArceOsCpuMask = api::task::AxCpuMask; -pub(crate) type ArceOsAxTaskExt = modules::ax_task::AxTaskExt; -pub(crate) type ArceOsAxTaskRef = modules::ax_task::AxTaskRef; -pub(crate) type ArceOsCurrentTask = modules::ax_task::CurrentTask; -pub(crate) type ArceOsTaskInner = modules::ax_task::TaskInner; -pub(crate) type ArceOsWaitQueue = modules::ax_task::WaitQueue; -pub(crate) type ArceOsWaitQueueHandle = api::task::AxWaitQueueHandle; -pub(crate) use modules::ax_task::TaskExt as ArceOsTaskExt; - -pub(crate) fn current_task() -> ArceOsCurrentTask { - modules::ax_task::current() -} - -pub(crate) fn spawn_task(task: ArceOsTaskInner) -> ArceOsAxTaskRef { - modules::ax_task::spawn_task(task) -} - -pub(crate) fn yield_now() { - thread::yield_now(); -} - -pub(crate) fn wait_queue_wait_until( - queue: &api::task::AxWaitQueueHandle, - condition: impl Fn() -> bool, -) { - api::task::ax_wait_queue_wait_until(queue, condition, None); -} - -pub(crate) fn wait_queue_wake(queue: &api::task::AxWaitQueueHandle, count: u32) { - api::task::ax_wait_queue_wake(queue, count); -} - -pub(crate) fn send_ipi(cpu_id: usize) { - if modules::ax_hal::percpu::this_cpu_id() == cpu_id { - return; - } - modules::ax_hal::irq::send_ipi( - modules::ax_hal::irq::ipi_irq(), - modules::ax_hal::irq::IpiTarget::Other { cpu_id }, - ); -} - -fn send_ipi_to_all_except_current(cpu_num: usize) { - if cpu_num <= 1 { - return; - } - let cpu_id = modules::ax_hal::percpu::this_cpu_id(); - modules::ax_hal::irq::send_ipi( - modules::ax_hal::irq::ipi_irq(), - modules::ax_hal::irq::IpiTarget::AllExceptCurrent { cpu_id, cpu_num }, - ); -} - -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqContext = modules::ax_hal::irq::IrqContext; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqError = modules::ax_hal::irq::IrqError; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqHandle = modules::ax_hal::irq::IrqHandle; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqId = modules::ax_hal::irq::IrqId; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqReturn = modules::ax_hal::irq::IrqReturn; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqSource = modules::ax_hal::irq::IrqSource; -#[cfg(target_arch = "x86_64")] -pub(crate) fn request_shared_irq( - irq: ArceOsIrqId, - handler: impl FnMut(ArceOsIrqContext) -> ArceOsIrqReturn + Send + 'static, -) -> Result { - modules::ax_hal::irq::request_shared_irq(irq, handler) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn make_irq_id(domain: u16, hwirq: u32) -> ArceOsIrqId { - modules::ax_hal::irq::IrqId::new( - modules::ax_hal::irq::IrqDomainId(domain), - modules::ax_hal::irq::HwIrq(hwirq), - ) -} - -#[cfg(all(target_arch = "x86_64", not(test)))] -pub(crate) fn set_irq_enable(irq: ArceOsIrqId, enabled: bool) -> Result<(), ArceOsIrqError> { - modules::ax_hal::irq::set_enable(irq, enabled) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn resolve_irq_source(source: ArceOsIrqSource) -> Result { - modules::ax_hal::irq::resolve_irq_source(source) -} - -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub(crate) fn host_fdt_bootarg() -> usize { - modules::ax_hal::dtb::get_bootarg() -} - -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub(crate) fn phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { - modules::ax_hal::mem::phys_to_virt(paddr) -} - -#[cfg(all( - any(feature = "fs", feature = "host-fs"), - any(target_arch = "x86_64", target_arch = "loongarch64") -))] -pub(crate) fn shutdown_host_filesystems() -> AxResult { - modules::ax_fs_ng::shutdown_filesystems()?; - let released = modules::ax_fs_ng::release_block_irqs_for_passthrough(); - if released != 0 { - info!("Released {released} host filesystem block IRQ registration(s) before passthrough"); - } - Ok(()) -} - -#[cfg(target_arch = "x86_64")] -impl HostConsole for ArceOsHost { - fn write_bytes(&self, bytes: &[u8]) { - modules::ax_hal::console::write_bytes(bytes); - } - - fn read_bytes(&self, bytes: &mut [u8]) -> usize { - modules::ax_hal::console::read_bytes(bytes) - } -} - -impl HostPlatform for ArceOsHost { - fn has_hardware_support(&self) -> bool { - CurrentArch::has_hardware_support() - } - - fn enable_virtualization_on_current_cpu(&self) -> AxResult { - crate::timer::init_percpu(); - crate::percpu::init_current_cpu()?; - crate::percpu::enable_current_cpu()?; - crate::percpu::mark_cpu_enabled(self.this_cpu_id()); - Ok(()) - } - - fn enable_virtualization_on_all_cpus(&self) -> AxResult { - static CORES: AtomicUsize = AtomicUsize::new(0); - - info!("Enabling hardware virtualization support on all cores..."); - CORES.store(0, Ordering::Release); - crate::percpu::reset_enabled_cpu_mask(); - - let cpu_count = self.cpu_count(); - let current_cpu = self.this_cpu_id(); - info!("Core {current_cpu} is initializing hardware virtualization support..."); - self.enable_virtualization_on_current_cpu()?; - info!("Hardware virtualization support enabled on core {current_cpu}"); - CORES.store(1, Ordering::Release); - - for cpu_id in 0..cpu_count { - if cpu_id == current_cpu { - continue; - } - let task = modules::ax_task::TaskInner::new( - move || { - let host = arceos_host(); - info!("Core {cpu_id} is initializing hardware virtualization support..."); - host.enable_virtualization_on_current_cpu() - .expect("failed to enable hardware virtualization"); - info!("Hardware virtualization support enabled on core {cpu_id}"); - let _ = CORES.fetch_add(1, Ordering::Release); - }, - alloc::format!("axvm-hv-init-{cpu_id}"), - modules::ax_task::default_task_stack_size(), - ); - task.set_cpumask(::CpuMask::one_shot(cpu_id)); - modules::ax_task::spawn_task(task); - if cpu_id != self.this_cpu_id() { - send_ipi(cpu_id); - } - } - - info!("Waiting for all cores to enable hardware virtualization..."); - let start = self.monotonic_time(); - let mut wait_rounds = 0usize; - while CORES.load(Ordering::Acquire) != cpu_count { - thread::yield_now(); - wait_rounds = wait_rounds.wrapping_add(1); - if wait_rounds.is_multiple_of(256) { - send_ipi_to_all_except_current(cpu_count); - } - if self.monotonic_time().saturating_sub(start) >= CPU_ENABLE_WAIT_TIMEOUT { - break; - } - } - CurrentArch::register_platform_irq_injector(); - let enabled_count = CORES.load(Ordering::Acquire); - if enabled_count == cpu_count { - info!("All cores have enabled hardware virtualization support."); - } else { - warn!( - "Only {enabled_count}/{cpu_count} cores enabled hardware virtualization before \ - timeout; continuing with host CPU mask {:#x}", - crate::percpu::enabled_cpu_mask() - ); - } - Ok(()) - } -} +pub(crate) use crate::arch::arceos::*; diff --git a/virtualization/axvm/src/host/mod.rs b/virtualization/axvm/src/host/mod.rs index 40b161d698..deb8952786 100644 --- a/virtualization/axvm/src/host/mod.rs +++ b/virtualization/axvm/src/host/mod.rs @@ -1,21 +1,14 @@ //! Internal host boundary used by the AxVM runtime. pub(crate) mod arceos; -#[cfg(target_arch = "aarch64")] -pub(crate) mod gic; -#[cfg(target_arch = "x86_64")] pub(crate) mod irq; pub(crate) mod paging; pub(crate) mod task; pub(crate) mod traits; -#[cfg(target_arch = "x86_64")] -pub(crate) mod x86_port; pub(crate) fn default_host() -> &'static arceos::ArceOsHost { arceos::arceos_host() } pub(crate) use paging::PagingHandler; -#[cfg(target_arch = "x86_64")] -pub(crate) use traits::HostConsole; -pub(crate) use traits::{HostCpu, HostMemory, HostPlatform, HostTime}; +pub(crate) use traits::{HostConsole, HostCpu, HostMemory, HostPlatform, HostTime}; diff --git a/virtualization/axvm/src/host/traits.rs b/virtualization/axvm/src/host/traits.rs index 8225612376..ba8e9d43d8 100644 --- a/virtualization/axvm/src/host/traits.rs +++ b/virtualization/axvm/src/host/traits.rs @@ -2,11 +2,6 @@ extern crate alloc; -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] use alloc::boxed::Box; use core::time::Duration; @@ -44,22 +39,15 @@ pub trait HostTime { type CancelToken: Copy + Send + Sync + 'static; /// Convert nanoseconds to hardware ticks. - #[cfg(target_arch = "x86_64")] fn nanos_to_ticks(&self, nanos: u64) -> u64; /// Read monotonic host time. fn monotonic_time(&self) -> Duration; /// Program the host one-shot timer. - #[cfg(not(target_arch = "loongarch64"))] fn set_oneshot_timer(&self, deadline_ns: u64); /// Register a VM timer callback. - #[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" - ))] fn register_timer( &self, deadline_ns: u64, @@ -67,7 +55,6 @@ pub trait HostTime { ) -> Self::CancelToken; /// Cancel a VM timer callback. - #[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] fn cancel_timer(&self, token: Self::CancelToken); } @@ -84,7 +71,6 @@ pub trait HostCpu { } /// Host console operations. -#[cfg(target_arch = "x86_64")] pub trait HostConsole { /// Write raw bytes to host console. fn write_bytes(&self, bytes: &[u8]); diff --git a/virtualization/axvm/src/irq/mod.rs b/virtualization/axvm/src/irq/mod.rs index 0d8930aa1b..43abba80f4 100644 --- a/virtualization/axvm/src/irq/mod.rs +++ b/virtualization/axvm/src/irq/mod.rs @@ -21,11 +21,7 @@ use axdevice::IrqResolver; use axdevice_base::{InterruptTriggerMode, IrqLine, IrqLineId, IrqSink}; use axvm_types::VMInterruptMode; -#[cfg(target_arch = "riscv64")] -pub(crate) mod riscv; - /// Host platform hook for registering the RISC-V physical IRQ injector. -#[cfg(target_arch = "riscv64")] #[ax_crate_interface::def_interface] pub trait RiscvPlatformIrqInjectorIf { /// Registers a callback that forwards a physical IRQ line into the current guest. @@ -35,14 +31,20 @@ pub trait RiscvPlatformIrqInjectorIf { fn set_virtual_irq_targets(cpu_id: usize, irq_sources: &[u32]); } -#[cfg(target_arch = "riscv64")] +#[expect( + dead_code, + reason = "the RISC-V architecture backend is not compiled for this target" +)] pub(crate) fn register_riscv_virtual_irq_injector(injector: fn(usize) -> bool) { ax_crate_interface::call_interface!(RiscvPlatformIrqInjectorIf::register_virtual_irq_injector( injector )); } -#[cfg(target_arch = "riscv64")] +#[expect( + dead_code, + reason = "the RISC-V architecture backend is not compiled for this target" +)] pub(crate) fn set_riscv_virtual_irq_targets(cpu_id: usize, irq_sources: &[u32]) { ax_crate_interface::call_interface!(RiscvPlatformIrqInjectorIf::set_virtual_irq_targets( cpu_id, diff --git a/virtualization/axvm/src/lib.rs b/virtualization/axvm/src/lib.rs index 3eea2fb8bf..28decb2270 100644 --- a/virtualization/axvm/src/lib.rs +++ b/virtualization/axvm/src/lib.rs @@ -41,6 +41,7 @@ use crate::arch::ArchOps; pub mod config; +pub use arch::platform::*; pub use ax_cpumask::CpuMask; /// Compatibility export for legacy/common normalized VM events. /// @@ -62,11 +63,6 @@ pub use manager::{ AxvmRuntime, current_vcpu_id, current_vm_id, get_vm_by_id, get_vm_list, inject_current_vcpu_interrupt, register_vm, }; -#[cfg(target_arch = "loongarch64")] -pub use runtime::loongarch_irq::{ - register_guest_irq_route as register_loongarch_guest_irq_route, - unregister_guest_irq_routes as unregister_loongarch_guest_irq_routes, -}; pub(crate) use task::{AsVCpuTask, VCpuTask}; pub use vm::{ AxVM, AxVMRef, FwCfgDeviceConfig, PreparedMemoryLayout, VMMemoryRegion, VcpuSnapshot, @@ -84,56 +80,3 @@ pub fn check_timer_events() { pub fn clean_dcache_range(addr: ax_memory_addr::VirtAddr, size: usize) { arch::CurrentArch::clean_dcache_range(addr, size); } - -/// Return the host FDT boot argument physical address. -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub fn host_fdt_bootarg() -> usize { - host::arceos::host_fdt_bootarg() -} - -/// Convert a host physical address into a host virtual address. -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { - host::arceos::phys_to_virt(paddr) -} - -/// Shut down ArceOS filesystems so guest passthrough can take ownership. -#[cfg(all( - any(feature = "fs", feature = "host-fs"), - any(target_arch = "x86_64", target_arch = "loongarch64") -))] -pub fn shutdown_host_filesystems() -> ax_errno::AxResult { - host::arceos::shutdown_host_filesystems() -} - -/// Register a native host IRQ as the source for one x86 guest IOAPIC GSI. -#[cfg(target_arch = "x86_64")] -pub fn register_x86_ioapic_irq_forwarding_route(guest_gsi: usize, host_irq: irq_framework::IrqId) { - runtime::register_x86_ioapic_irq_forwarding_route(guest_gsi, host_irq); -} - -/// Register a native host IRQ and trigger mode as the source for one x86 guest -/// IOAPIC GSI. -#[cfg(target_arch = "x86_64")] -pub fn register_x86_ioapic_irq_forwarding_route_with_trigger( - guest_gsi: usize, - host_irq: irq_framework::IrqId, - trigger: InterruptTriggerMode, -) { - runtime::register_x86_ioapic_irq_forwarding_route_with_trigger(guest_gsi, host_irq, trigger); -} - -/// Register a callback to activate one x86 guest IOAPIC GSI after the guest has -/// programmed a usable virtual IOAPIC route for it. -#[cfg(target_arch = "x86_64")] -pub fn register_x86_ioapic_irq_forwarding_activator(guest_gsi: usize, activator: fn()) { - runtime::register_x86_ioapic_irq_forwarding_activator(guest_gsi, activator); -} diff --git a/virtualization/axvm/src/manager.rs b/virtualization/axvm/src/manager.rs index d0aa97b20f..69ae065928 100644 --- a/virtualization/axvm/src/manager.rs +++ b/virtualization/axvm/src/manager.rs @@ -54,7 +54,6 @@ pub fn get_vm_list() -> Vec { } /// Run an operation with a VM selected from the process-wide runtime registry. -#[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] pub(crate) fn with_vm(vm_id: VMId, f: F) -> Option where F: FnOnce(&AxVMRef) -> R, @@ -64,7 +63,6 @@ where } /// Return the active-vCPU mask for a VM. -#[cfg(target_arch = "x86_64")] pub(crate) fn active_vcpu_mask(vm_id: VMId) -> Option { with_vm(vm_id, |vm| { let vcpu_num = vm.vcpu_num(); @@ -77,13 +75,15 @@ pub(crate) fn active_vcpu_mask(vm_id: VMId) -> Option { } /// Inject a virtual interrupt into a VM's vCPU. -#[cfg(target_arch = "x86_64")] pub(crate) fn inject_interrupt(vm_id: VMId, vcpu_id: usize, vector: usize) -> AxResult { crate::runtime::vcpus::queue_interrupt(vm_id, vcpu_id, vector) } /// Inject a virtual interrupt into a VM's vCPU. -#[cfg(target_arch = "loongarch64")] +#[expect( + dead_code, + reason = "only the LoongArch IRQ backend injects external VM interrupts" +)] pub(crate) fn inject_vm_vcpu_interrupt(vm_id: VMId, vcpu_id: usize, vector: usize) -> AxResult { use crate::AsVCpuTask; diff --git a/virtualization/axvm/src/runtime/mod.rs b/virtualization/axvm/src/runtime/mod.rs index 884c36fd93..1e30038dc8 100644 --- a/virtualization/axvm/src/runtime/mod.rs +++ b/virtualization/axvm/src/runtime/mod.rs @@ -14,18 +14,11 @@ pub(crate) mod hvc; mod ivc; - -#[cfg(target_arch = "loongarch64")] -pub mod loongarch_irq; pub(crate) mod vcpus; -#[cfg(target_arch = "x86_64")] -pub(crate) mod x86_irq; use core::sync::atomic::{AtomicUsize, Ordering}; use ax_errno::{AxResult, ax_err, ax_err_type}; -#[cfg(target_arch = "x86_64")] -use axvm_types::InterruptTriggerMode; use crate::{StopReason, VmStatus}; @@ -132,26 +125,6 @@ pub fn register_vm(vm: VMRef) -> bool { crate::manager::push_existing_vm(vm) } -/// Register a native host IRQ as the source for one x86 guest IOAPIC GSI. -#[cfg(target_arch = "x86_64")] -pub(crate) fn register_x86_ioapic_irq_forwarding_route( - guest_gsi: usize, - host_irq: irq_framework::IrqId, -) { - x86_irq::register_ioapic_irq_forwarding_route(guest_gsi, host_irq); -} - -/// Register a native host IRQ and trigger mode as the source for one x86 guest -/// IOAPIC GSI. -#[cfg(target_arch = "x86_64")] -pub(crate) fn register_x86_ioapic_irq_forwarding_route_with_trigger( - guest_gsi: usize, - host_irq: irq_framework::IrqId, - trigger: InterruptTriggerMode, -) { - x86_irq::register_ioapic_irq_forwarding_route_with_trigger(guest_gsi, host_irq, trigger); -} - #[cfg(test)] mod tests { use super::*; @@ -172,10 +145,3 @@ mod tests { } } } - -/// Register a callback to activate one x86 guest IOAPIC GSI after the guest has -/// programmed a usable virtual IOAPIC route for it. -#[cfg(target_arch = "x86_64")] -pub(crate) fn register_x86_ioapic_irq_forwarding_activator(guest_gsi: usize, activator: fn()) { - x86_irq::register_ioapic_irq_forwarding_activator(guest_gsi, activator); -} diff --git a/virtualization/axvm/src/runtime/vcpus.rs b/virtualization/axvm/src/runtime/vcpus.rs index d26a538966..8466290283 100644 --- a/virtualization/axvm/src/runtime/vcpus.rs +++ b/virtualization/axvm/src/runtime/vcpus.rs @@ -17,13 +17,11 @@ use alloc::format; use ax_errno::{AxResult, ax_err_type}; use crate::{ - AsVCpuTask, StopReason, VCpuTask, VmStatus, + AsVCpuTask, GuestPhysAddr, StopReason, VCpuTask, VmStatus, VmVcpuState, arch::{ArchOps, CurrentArch, VcpuRunAction}, runtime::{VCpuRef, VMRef, sub_running_vm_count}, vm::VmRuntimeHandle, }; -#[cfg(not(target_arch = "x86_64"))] -use crate::{GuestPhysAddr, VmVcpuState}; const KERNEL_STACK_SIZE: usize = 0x40000; // 256 KiB @@ -105,7 +103,10 @@ pub(crate) fn queue_interrupt(vm_id: usize, vcpu_id: usize, vector: usize) -> Ax Ok(()) } -#[cfg(target_arch = "loongarch64")] +#[expect( + dead_code, + reason = "only the LoongArch IRQ backend queues physical interrupts" +)] pub(crate) fn queue_external_interrupt( vm_id: usize, vcpu_id: usize, @@ -190,7 +191,10 @@ fn mark_vcpu_running(vm: &VMRef) { /// * `vcpu_id` - The ID of the VCpu to be booted. /// * `entry_point` - The entry point of the VCpu. /// * `arg` - The argument to be passed to the VCpu. -#[cfg(not(target_arch = "x86_64"))] +#[expect( + dead_code, + reason = "only non-x86 guest firmware boots secondary vCPUs" +)] pub(crate) fn vcpu_on( vm: VMRef, vcpu_id: usize, @@ -220,7 +224,10 @@ pub(crate) fn vcpu_on( Ok(()) } -#[cfg(not(target_arch = "x86_64"))] +#[expect( + dead_code, + reason = "only non-x86 guest firmware boots secondary vCPUs" +)] pub(crate) fn alloc_vcpu_task(vm: &VMRef, vcpu: VCpuRef) -> crate::AxTaskRef { crate::host::task::spawn_task(build_vcpu_task(vm, vcpu)) } diff --git a/virtualization/axvm/src/timer.rs b/virtualization/axvm/src/timer.rs index da674abc30..0796a566df 100644 --- a/virtualization/axvm/src/timer.rs +++ b/virtualization/axvm/src/timer.rs @@ -3,18 +3,10 @@ extern crate alloc; use alloc::boxed::Box; -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] -use core::sync::atomic::{AtomicUsize, Ordering}; -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] -use core::time::Duration; +use core::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; use ax_kspin::SpinNoIrq; use ax_lazyinit::LazyInit; @@ -22,33 +14,19 @@ use ax_timer_list::{TimeValue, TimerEvent, TimerList}; use crate::host::{HostTime, default_host}; -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] static TOKEN: AtomicUsize = AtomicUsize::new(0); struct VmTimerEvent { - #[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] token: usize, callback: Box, } impl VmTimerEvent { - #[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" - ))] fn new(token: usize, callback: F) -> Self where F: FnOnce(TimeValue) + Send + 'static, { - #[cfg(not(any(target_arch = "x86_64", target_arch = "loongarch64")))] - let _ = token; Self { - #[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] token, callback: Box::new(callback), } @@ -64,11 +42,6 @@ impl TimerEvent for VmTimerEvent { #[ax_percpu::def_percpu] static TIMER_LIST: LazyInit>> = LazyInit::new(); -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] pub(crate) fn register_timer( deadline_ns: u64, callback: Box, @@ -89,7 +62,6 @@ pub(crate) fn register_timer( token } -#[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] pub(crate) fn cancel_timer(token: usize) { let next_deadline = { // SAFETY: The timer list is initialized for each CPU before VM timer @@ -120,9 +92,6 @@ pub(crate) fn check_events() { } fn rearm_host_timer(next_deadline: Option) { - #[cfg(target_arch = "loongarch64")] - let _ = next_deadline; - #[cfg(not(target_arch = "loongarch64"))] if let Some(deadline) = next_deadline { default_host().set_oneshot_timer(deadline.as_nanos() as u64); } @@ -134,6 +103,5 @@ pub(crate) fn init_percpu() { // CPU can register VM timers. let timer_list = unsafe { TIMER_LIST.current_ref_mut_raw() }; timer_list.init_once(SpinNoIrq::new(TimerList::new())); - #[cfg(target_arch = "loongarch64")] - ax_std::os::arceos::modules::ax_task::register_timer_callback(|_| check_events()); + crate::arch::register_timer_callback(); } diff --git a/virtualization/axvm/src/vcpu.rs b/virtualization/axvm/src/vcpu.rs index 7683e0f3af..626533f0c1 100644 --- a/virtualization/axvm/src/vcpu.rs +++ b/virtualization/axvm/src/vcpu.rs @@ -19,8 +19,6 @@ use core::{cell::UnsafeCell, mem::MaybeUninit}; use ax_errno::{AxResult, ax_err}; use ax_kspin::SpinNoIrq as Mutex; -#[cfg(target_arch = "x86_64")] -use axvm_types::InterruptTriggerMode; use axvm_types::{ GuestPhysAddr, NestedPagingConfig, VCpuId, VMId, VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState, }; @@ -201,7 +199,10 @@ impl AxVCpu { } /// Sets the guest entry point. - #[cfg(not(target_arch = "x86_64"))] + #[expect( + dead_code, + reason = "only non-x86 guest firmware updates secondary vCPU entries" + )] pub fn set_entry(&self, entry: GuestPhysAddr) -> AxResult { self.get_arch_vcpu().set_entry(entry) } @@ -216,17 +217,6 @@ impl AxVCpu { self.get_arch_vcpu().inject_interrupt(vector) } - /// Injects an interrupt with trigger-mode metadata. - #[cfg(target_arch = "x86_64")] - pub fn inject_interrupt_with_trigger( - &self, - vector: usize, - trigger: InterruptTriggerMode, - ) -> AxResult { - self.get_arch_vcpu() - .inject_interrupt_with_trigger(vector, trigger) - } - /// Sets the guest return value. pub fn set_return_value(&self, val: usize) { self.get_arch_vcpu().set_return_value(val); diff --git a/virtualization/axvm/src/vm.rs b/virtualization/axvm/src/vm.rs index 41408cf656..fadc984ef9 100644 --- a/virtualization/axvm/src/vm.rs +++ b/virtualization/axvm/src/vm.rs @@ -171,7 +171,10 @@ impl VmRuntimeHandle { Ok(task.cpu_id() as usize) } - #[cfg(target_arch = "loongarch64")] + #[expect( + dead_code, + reason = "only the LoongArch IRQ backend queues physical interrupts" + )] pub(crate) fn queue_external_interrupt( &self, vcpu_id: usize, @@ -841,7 +844,6 @@ impl AxVM { Ok(()) } - #[cfg(not(target_arch = "aarch64"))] pub(crate) fn handle_nested_page_fault( &self, addr: GuestPhysAddr, @@ -857,7 +859,6 @@ impl AxVM { .unwrap_or(false) } - #[cfg(not(target_arch = "aarch64"))] fn debug_nested_page_fault( vm_id: usize, resources: &AxVMResources, diff --git a/virtualization/axvm/src/vm/prepare.rs b/virtualization/axvm/src/vm/prepare.rs index c609e1befd..2b236bd253 100644 --- a/virtualization/axvm/src/vm/prepare.rs +++ b/virtualization/axvm/src/vm/prepare.rs @@ -20,23 +20,12 @@ impl AxVM { let mut factories = DeviceFactoryRegistry::new(); register_builtin_factories(&mut factories)?; let interrupt_mode = self.interrupt_mode(); - #[cfg(target_arch = "riscv64")] - let interrupt_fabric = { - let machine = self.machine.lock(); - let resources = machine.resources().ok_or_else(|| { - ax_err_type!( - BadState, - "VM resources are not available for RISC-V IRQ setup" - ) - })?; - crate::irq::riscv::configure( - &mut factories, - interrupt_mode, - resources.config.emu_devices(), - )? - }; - #[cfg(not(target_arch = "riscv64"))] - let interrupt_fabric = InterruptFabric::new(interrupt_mode); + let emulated_devices = self.with_config(|config| config.emu_devices().clone()); + let interrupt_fabric = crate::arch::configure_interrupt_fabric( + &mut factories, + interrupt_mode, + &emulated_devices, + )?; self.prepare_with_factories(&factories, interrupt_fabric) } diff --git a/virtualization/axvm/src/vm/prepare/address_space.rs b/virtualization/axvm/src/vm/prepare/address_space.rs index 9c8d76ca6e..2cc356dcd6 100644 --- a/virtualization/axvm/src/vm/prepare/address_space.rs +++ b/virtualization/axvm/src/vm/prepare/address_space.rs @@ -5,10 +5,6 @@ use alloc::vec::Vec; use ax_errno::AxResult; use axdevice::AxVmDevices; use axdevice_base::Resource; -#[cfg(all(target_arch = "x86_64", feature = "vmx"))] -use axvm_types::GuestPhysAddr; -#[cfg(all(target_arch = "x86_64", feature = "vmx"))] -use x86_vcpu::X86_APIC_ACCESS_GPA; use super::super::{AxVM, AxVMResources, VM_ASPACE_BASE, VM_ASPACE_SIZE}; use crate::layout::{GuestOwnedRegion, VmRegionKind, build_address_layout}; @@ -53,15 +49,7 @@ pub(super) fn map_guest_address_space( } resources.address_layout = Some(address_layout); - #[cfg(all(target_arch = "x86_64", feature = "vmx"))] - resources.address_space.map_linear( - GuestPhysAddr::from(X86_APIC_ACCESS_GPA), - crate::arch::x86_apic_access_page_addr(), - ax_memory_addr::PAGE_SIZE_4K, - axvm_types::MappingFlags::DEVICE - | axvm_types::MappingFlags::READ - | axvm_types::MappingFlags::WRITE, - )?; + crate::arch::map_arch_address_space(&mut resources.address_space)?; Ok(()) } @@ -93,12 +81,7 @@ fn guest_owned_regions(resources: &AxVMResources) -> Vec { }), ); - #[cfg(all(target_arch = "x86_64", feature = "vmx"))] - regions.push(GuestOwnedRegion::new( - X86_APIC_ACCESS_GPA, - ax_memory_addr::PAGE_SIZE_4K, - VmRegionKind::Reserved, - )); + crate::arch::append_arch_owned_regions(&mut regions); regions } diff --git a/virtualization/axvm/src/vm/prepare/devices.rs b/virtualization/axvm/src/vm/prepare/devices.rs index 23700f5bc8..3db1319ffe 100644 --- a/virtualization/axvm/src/vm/prepare/devices.rs +++ b/virtualization/axvm/src/vm/prepare/devices.rs @@ -1,20 +1,9 @@ //! Device construction for VM preparation. -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -use alloc::{format, sync::Arc}; - use ax_errno::AxResult; -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -use ax_errno::ax_err_type; use axdevice::{AxVmDeviceConfig, AxVmDevices, DeviceBuildContext, DeviceFactoryRegistry}; -#[cfg(target_arch = "aarch64")] -use axdevice_base::DeviceRegistry as _; -#[cfg(target_arch = "x86_64")] -use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; use super::super::{AxVM, AxVMResources}; -#[cfg(target_arch = "aarch64")] -use crate::config::VMInterruptMode; use crate::irq::InterruptFabric; pub(super) struct PreparedDevices { @@ -37,24 +26,7 @@ impl PreparedDevices { &build_context, )?; - #[cfg(target_arch = "x86_64")] - for port in resources.config.pass_through_ports() { - let passthrough = Arc::new(crate::host::x86_port::HostPortPassthrough::new( - port.base, - port.length, - )?); - let range = passthrough.address_range(); - debug!( - "PT port region: [{:#x}~{:#x}]", - range.start.number(), - range.end.number(), - ); - devices - .register(PortDeviceAdapter::from_arc(passthrough)) - .map_err(|err| ax_err_type!(InvalidInput, format!("register PT port: {err:?}")))?; - } - - Self::register_arch_devices(vm, resources, &mut devices)?; + crate::arch::register_arch_devices(vm, &resources.config, &mut devices)?; vm.add_special_emulated_devices(&mut devices)?; Ok(Self { devices }) } @@ -66,63 +38,4 @@ impl PreparedDevices { pub(super) fn into_inner(self) -> AxVmDevices { self.devices } - - #[cfg(target_arch = "aarch64")] - fn register_arch_devices( - vm: &AxVM, - resources: &AxVMResources, - devices: &mut AxVmDevices, - ) -> AxResult { - let passthrough = resources.config.interrupt_mode() == VMInterruptMode::Passthrough; - if passthrough { - let spis = resources.config.pass_through_spis(); - let cpu_id = vm.id() - 1; // FIXME: get the real CPU id. - let mut gicd_found = false; - - for device in devices.devices() { - if let Some(gicd) = device.as_any().downcast_ref::() { - debug!("VGicD found, assigning SPIs..."); - - for spi in spis { - gicd.assign_irq(*spi + 32, cpu_id, (0, 0, 0, cpu_id as _)) - } - - gicd_found = true; - break; - } - } - - if !gicd_found { - warn!("Failed to assign SPIs: No VGicD found in device list"); - } - } else { - for dev in axdevice::create_vtimer_devices() { - devices - .register(Arc::from(dev) as Arc) - .map_err(|e| ax_err_type!(InvalidInput, format!("register vtimer: {e:?}")))?; - } - } - Ok(()) - } - - #[cfg(target_arch = "x86_64")] - fn register_arch_devices( - _vm: &AxVM, - resources: &AxVMResources, - devices: &mut AxVmDevices, - ) -> AxResult { - for config in resources.config.emu_devices() { - crate::arch::register_x86_arch_device(config, devices)?; - } - Ok(()) - } - - #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] - fn register_arch_devices( - _vm: &AxVM, - _resources: &AxVMResources, - _devices: &mut AxVmDevices, - ) -> AxResult { - Ok(()) - } } diff --git a/virtualization/axvm/tests/arch_boundary_contract.rs b/virtualization/axvm/tests/arch_boundary_contract.rs index d7861d0d9f..b1ff37c57e 100644 --- a/virtualization/axvm/tests/arch_boundary_contract.rs +++ b/virtualization/axvm/tests/arch_boundary_contract.rs @@ -45,3 +45,70 @@ fn runtime_vcpu_loop_only_consumes_scheduler_actions() { ); } } + +#[test] +fn production_sources_keep_architecture_cfg_inside_arch_module() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let violations = find_target_arch_cfg_outside_arch(&source_root, &source_root); + + assert!( + violations.is_empty(), + "target_arch must stay inside src/arch; found: {}", + violations.join(", ") + ); +} + +#[test] +fn arch_root_contains_only_architectures_and_shared_dispatch_modules() { + let arch_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/arch"); + let mut unexpected_entries = std::fs::read_dir(&arch_root) + .expect("AxVM architecture directory must be readable") + .map(|entry| entry.expect("AxVM architecture entry must be readable")) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| { + !matches!( + name.as_str(), + "aarch64" | "loongarch64" | "riscv64" | "x86_64" | "mod.rs" | "npt.rs" + ) + }) + .collect::>(); + unexpected_entries.sort(); + + assert!( + unexpected_entries.is_empty(), + "arch root must contain only architecture directories and shared dispatch modules; found: \ + {}", + unexpected_entries.join(", ") + ); +} + +fn find_target_arch_cfg_outside_arch( + source_root: &std::path::Path, + directory: &std::path::Path, +) -> Vec { + let mut violations = Vec::new(); + for entry in std::fs::read_dir(directory).expect("AxVM source directory must be readable") { + let entry = entry.expect("AxVM source directory entry must be readable"); + let path = entry.path(); + if path.is_dir() { + if path != source_root.join("arch") { + violations.extend(find_target_arch_cfg_outside_arch(source_root, &path)); + } + continue; + } + + if path.extension().is_some_and(|extension| extension == "rs") + && std::fs::read_to_string(&path) + .expect("AxVM source file must be readable") + .contains("target_arch") + { + violations.push( + path.strip_prefix(source_root) + .expect("source path must be below src") + .display() + .to_string(), + ); + } + } + violations +} From 4e34aeaf19010aea0a4713fd6ad5ed00bfca3fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Fri, 10 Jul 2026 14:32:11 +0800 Subject: [PATCH 2/5] refactor(axvm): organize architecture capabilities --- .../axvm/src/arch/aarch64/boot/fdt/create.rs | 6 +- .../axvm/src/arch/aarch64/boot/fdt/mod.rs | 14 +- .../axvm/src/arch/aarch64/boot/fdt/parser.rs | 2 +- .../axvm/src/arch/aarch64/capabilities.rs | 79 ++ virtualization/axvm/src/arch/aarch64/gic.rs | 4 +- .../axvm/src/arch/aarch64/images.rs | 33 + virtualization/axvm/src/arch/aarch64/ipi.rs | 90 ++ virtualization/axvm/src/arch/aarch64/mod.rs | 94 +- virtualization/axvm/src/arch/aarch64/npt.rs | 2 +- .../boot/fdt}/guest_firmware_dtb.rs | 2 +- .../boot/fdt}/mod.rs | 0 .../boot/fdt}/property.rs | 0 .../loongarch64/boot/guest_platform/mod.rs | 2 - .../{guest_platform/loongarch64 => }/mod.rs | 161 ++- .../{guest_platform/loongarch64 => }/probe.rs | 0 .../loongarch64 => }/resources.rs | 0 .../axvm/src/arch/loongarch64/capabilities.rs | 26 + .../axvm/src/arch/loongarch64/idle.rs | 26 + .../axvm/src/arch/loongarch64/irq.rs | 27 +- .../axvm/src/arch/loongarch64/mod.rs | 70 +- .../axvm/src/arch/loongarch64/npt.rs | 8 +- virtualization/axvm/src/arch/mod.rs | 874 +----------- .../axvm/src/arch/riscv64/capabilities.rs | 42 + .../axvm/src/arch/riscv64/images.rs | 33 + virtualization/axvm/src/arch/riscv64/mod.rs | 75 +- virtualization/axvm/src/arch/riscv64/npt.rs | 2 +- .../boot/{images/x86 => }/boot_params.rs | 6 +- .../axvm/src/arch/x86_64/boot/images/mod.rs | 1185 ----------------- .../src/arch/x86_64/boot/images/x86/mod.rs | 8 - .../x86_64/boot/{images/x86 => }/linux.rs | 0 .../boot/{images/x86 => }/linux_boot.rs | 6 +- .../axvm/src/arch/x86_64/boot/mod.rs | 537 ++++++++ .../x86_64/boot/{images/x86 => }/mptable.rs | 18 +- .../x86_64/boot/{images/x86 => }/multiboot.rs | 0 .../axvm/src/arch/x86_64/capabilities.rs | 64 + virtualization/axvm/src/arch/x86_64/exit.rs | 78 ++ virtualization/axvm/src/arch/x86_64/host.rs | 385 ------ .../{host/irq.rs => arch/x86_64/host_irq.rs} | 24 +- virtualization/axvm/src/arch/x86_64/irq.rs | 20 +- virtualization/axvm/src/arch/x86_64/mod.rs | 144 +- virtualization/axvm/src/arch/x86_64/npt.rs | 8 +- .../axvm/src/architecture/capabilities.rs | 75 ++ .../axvm/src/architecture/cpu_up.rs | 67 + virtualization/axvm/src/architecture/exit.rs | 58 + virtualization/axvm/src/architecture/mod.rs | 16 + virtualization/axvm/src/architecture/ops.rs | 244 ++++ .../axvm/src/architecture/sysreg.rs | 39 + virtualization/axvm/src/architecture/types.rs | 91 ++ .../src/{arch/aarch64 => }/boot/fdt/device.rs | 0 virtualization/axvm/src/boot/fdt/mod.rs | 6 + .../src/{arch/aarch64 => }/boot/fdt/print.rs | 0 .../src/{arch/aarch64 => }/boot/fdt/tree.rs | 1 - .../{arch/aarch64 => }/boot/fdt/tree_tests.rs | 0 .../axvm/src/boot/guest_platform/mod.rs | 5 + .../{arch/x86_64 => }/boot/images/linux.rs | 0 virtualization/axvm/src/boot/images/mod.rs | 329 +++++ virtualization/axvm/src/boot/mod.rs | 6 +- virtualization/axvm/src/host/arceos.rs | 263 +++- virtualization/axvm/src/host/mod.rs | 3 +- virtualization/axvm/src/host/traits.rs | 15 - virtualization/axvm/src/lib.rs | 2 + virtualization/axvm/src/{arch => }/npt.rs | 0 virtualization/axvm/src/runtime/vcpus.rs | 12 +- .../axvm/tests/arch_boundary_contract.rs | 42 +- 64 files changed, 2770 insertions(+), 2659 deletions(-) create mode 100644 virtualization/axvm/src/arch/aarch64/capabilities.rs create mode 100644 virtualization/axvm/src/arch/aarch64/images.rs create mode 100644 virtualization/axvm/src/arch/aarch64/ipi.rs rename virtualization/axvm/src/arch/{aarch64/boot/fdt/loongarch64 => loongarch64/boot/fdt}/guest_firmware_dtb.rs (99%) rename virtualization/axvm/src/arch/{aarch64/boot/fdt/loongarch64 => loongarch64/boot/fdt}/mod.rs (100%) rename virtualization/axvm/src/arch/{aarch64/boot/fdt/loongarch64 => loongarch64/boot/fdt}/property.rs (100%) delete mode 100644 virtualization/axvm/src/arch/loongarch64/boot/guest_platform/mod.rs rename virtualization/axvm/src/arch/loongarch64/boot/{guest_platform/loongarch64 => }/mod.rs (55%) rename virtualization/axvm/src/arch/loongarch64/boot/{guest_platform/loongarch64 => }/probe.rs (100%) rename virtualization/axvm/src/arch/loongarch64/boot/{guest_platform/loongarch64 => }/resources.rs (100%) create mode 100644 virtualization/axvm/src/arch/loongarch64/capabilities.rs create mode 100644 virtualization/axvm/src/arch/loongarch64/idle.rs create mode 100644 virtualization/axvm/src/arch/riscv64/capabilities.rs create mode 100644 virtualization/axvm/src/arch/riscv64/images.rs rename virtualization/axvm/src/arch/x86_64/boot/{images/x86 => }/boot_params.rs (99%) delete mode 100644 virtualization/axvm/src/arch/x86_64/boot/images/mod.rs delete mode 100644 virtualization/axvm/src/arch/x86_64/boot/images/x86/mod.rs rename virtualization/axvm/src/arch/x86_64/boot/{images/x86 => }/linux.rs (100%) rename virtualization/axvm/src/arch/x86_64/boot/{images/x86 => }/linux_boot.rs (98%) create mode 100644 virtualization/axvm/src/arch/x86_64/boot/mod.rs rename virtualization/axvm/src/arch/x86_64/boot/{images/x86 => }/mptable.rs (92%) rename virtualization/axvm/src/arch/x86_64/boot/{images/x86 => }/multiboot.rs (100%) create mode 100644 virtualization/axvm/src/arch/x86_64/capabilities.rs create mode 100644 virtualization/axvm/src/arch/x86_64/exit.rs delete mode 100644 virtualization/axvm/src/arch/x86_64/host.rs rename virtualization/axvm/src/{host/irq.rs => arch/x86_64/host_irq.rs} (71%) create mode 100644 virtualization/axvm/src/architecture/capabilities.rs create mode 100644 virtualization/axvm/src/architecture/cpu_up.rs create mode 100644 virtualization/axvm/src/architecture/exit.rs create mode 100644 virtualization/axvm/src/architecture/mod.rs create mode 100644 virtualization/axvm/src/architecture/ops.rs create mode 100644 virtualization/axvm/src/architecture/sysreg.rs create mode 100644 virtualization/axvm/src/architecture/types.rs rename virtualization/axvm/src/{arch/aarch64 => }/boot/fdt/device.rs (100%) create mode 100644 virtualization/axvm/src/boot/fdt/mod.rs rename virtualization/axvm/src/{arch/aarch64 => }/boot/fdt/print.rs (100%) rename virtualization/axvm/src/{arch/aarch64 => }/boot/fdt/tree.rs (99%) rename virtualization/axvm/src/{arch/aarch64 => }/boot/fdt/tree_tests.rs (100%) create mode 100644 virtualization/axvm/src/boot/guest_platform/mod.rs rename virtualization/axvm/src/{arch/x86_64 => }/boot/images/linux.rs (100%) create mode 100644 virtualization/axvm/src/boot/images/mod.rs rename virtualization/axvm/src/{arch => }/npt.rs (100%) diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs index c9e4a25c51..b9254f82bd 100644 --- a/virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs +++ b/virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs @@ -362,8 +362,10 @@ mod tests { use fdt_edit::{Fdt, Node, Property}; use fdt_raw::RegInfo; - use super::{cpu_node_id, initrd_range_from_image_config, need_cpu_node}; - use crate::{GuestPhysAddr, boot::fdt::tree::sanitize_bootargs, config::RamdiskInfo}; + use super::{ + super::tree::sanitize_bootargs, cpu_node_id, initrd_range_from_image_config, need_cpu_node, + }; + use crate::{GuestPhysAddr, config::RamdiskInfo}; fn prop_u32(name: &str, value: u32) -> Property { let mut prop = Property::new(name, alloc::vec![]); diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs index bf3e63d645..396c26fac5 100644 --- a/virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs +++ b/virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs @@ -20,21 +20,23 @@ #[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] mod create; #[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] +#[path = "../../../../boot/fdt/device.rs"] mod device; -#[cfg(target_arch = "loongarch64")] -pub(crate) mod loongarch64; #[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] mod parser; +#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] +#[path = "../../../../boot/fdt/print.rs"] mod print; #[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] +#[path = "../../../../boot/fdt/tree.rs"] mod tree; #[cfg(test)] +#[path = "../../../../boot/fdt/tree_tests.rs"] mod tree_tests; #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] use alloc::format; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] use alloc::vec::Vec; #[cfg(any( @@ -67,13 +69,11 @@ use crate::boot::BootImageProvider; use crate::config::AxVMConfig; /// Guest DTB artifact produced or patched by the monitor before AxVM owns it. -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] #[derive(Debug, Clone)] pub struct GuestDtbImage { bytes: Vec, } -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] impl GuestDtbImage { pub fn new(bytes: Vec) -> Self { Self { bytes } @@ -87,7 +87,7 @@ impl GuestDtbImage { /// Initialize LoongArch guest firmware resource handling. #[cfg(target_arch = "loongarch64")] pub fn init_guest_boot_resources() { - crate::boot::guest_platform::loongarch64::init(); + crate::arch::guest_platform::init(); } #[cfg(target_arch = "loongarch64")] @@ -95,7 +95,7 @@ fn handle_uefi_fdt_operations( vm_config: &mut AxVMConfig, vm_create_config: &mut AxVMCrateConfig, ) -> AxResult { - crate::boot::guest_platform::loongarch64::prepare_uefi_fdt_config(vm_config, vm_create_config) + crate::arch::guest_platform::prepare_uefi_fdt_config(vm_config, vm_create_config) } #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs index fcfa16172a..1c7677c502 100644 --- a/virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs +++ b/virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs @@ -28,7 +28,7 @@ use axvmconfig::{ use fdt_edit::{Fdt, Node, NodeType, PciRange, PciSpace}; #[cfg(target_arch = "aarch64")] -use crate::boot::fdt::create::update_cpu_node; +use super::create::update_cpu_node; use crate::{MappingFlags, config::AxVMConfig}; const PAGE_SIZE_4K: usize = 0x1000; diff --git a/virtualization/axvm/src/arch/aarch64/capabilities.rs b/virtualization/axvm/src/arch/aarch64/capabilities.rs new file mode 100644 index 0000000000..364defd376 --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/capabilities.rs @@ -0,0 +1,79 @@ +//! AArch64 implementations of AxVM platform capability hooks. + +use alloc::{format, sync::Arc}; + +use ax_errno::{AxResult, ax_err_type}; +use axdevice_base::DeviceRegistry as _; +use axvm_types::VMInterruptMode; + +use super::Aarch64Arch; +use crate::architecture::{ + AddressSpacePlatform, BootImagePlatform, DevicePlatform, HostTimePlatform, +}; + +impl DevicePlatform for Aarch64Arch { + fn register_devices( + vm: &crate::AxVM, + config: &crate::config::AxVMConfig, + devices: &mut axdevice::AxVmDevices, + ) -> AxResult { + if config.interrupt_mode() == VMInterruptMode::Passthrough { + assign_passthrough_spis(vm, config, devices); + } else { + register_virtual_timers(devices)?; + } + Ok(()) + } +} + +impl AddressSpacePlatform for Aarch64Arch {} + +impl HostTimePlatform for Aarch64Arch {} + +impl BootImagePlatform for Aarch64Arch { + fn load_guest_dtb( + loader: &crate::boot::images::ImageLoaderCore<'_>, + dtb: &crate::boot::fdt::GuestDtbImage, + ) -> AxResult { + let bytes = dtb.as_bytes(); + let source = core::ptr::NonNull::new(bytes.as_ptr() as *mut u8) + .ok_or_else(|| ax_err_type!(InvalidData, "Guest DTB pointer is null"))?; + crate::boot::fdt::update_fdt(source, bytes.len(), loader.vm.clone(), &loader.config) + } +} + +pub fn host_fdt_bootarg() -> usize { + ax_std::os::arceos::modules::ax_hal::dtb::get_bootarg() +} + +pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + ax_std::os::arceos::modules::ax_hal::mem::phys_to_virt(paddr) +} + +fn assign_passthrough_spis( + vm: &crate::AxVM, + config: &crate::config::AxVMConfig, + devices: &axdevice::AxVmDevices, +) { + let cpu_id = vm.id() - 1; // FIXME: get the real CPU id. + let Some(gicd) = devices + .devices() + .find_map(|device| device.as_any().downcast_ref::()) + else { + warn!("Failed to assign SPIs: No VGicD found in device list"); + return; + }; + + for spi in config.pass_through_spis() { + gicd.assign_irq(*spi + 32, cpu_id, (0, 0, 0, cpu_id as _)); + } +} + +fn register_virtual_timers(devices: &mut axdevice::AxVmDevices) -> AxResult { + for device in axdevice::create_vtimer_devices() { + devices + .register(Arc::from(device) as Arc) + .map_err(|err| ax_err_type!(InvalidInput, format!("register vtimer: {err:?}")))?; + } + Ok(()) +} diff --git a/virtualization/axvm/src/arch/aarch64/gic.rs b/virtualization/axvm/src/arch/aarch64/gic.rs index 1c0e53d869..5f97b8c5c8 100644 --- a/virtualization/axvm/src/arch/aarch64/gic.rs +++ b/virtualization/axvm/src/arch/aarch64/gic.rs @@ -6,7 +6,7 @@ use arm_gic_driver::v3::{ }; use ax_memory_addr::{PhysAddr, VirtAddr}; -use super::{HostMemory, arceos, default_host}; +use crate::host::{HostMemory, default_host}; fn with_gic(f: impl FnOnce(&mut rdif_intc::Intc) -> T) -> T { let mut gic = rdrive::get_one::() @@ -140,7 +140,7 @@ pub(crate) fn handle_current_irq() -> Option { // AArch64 ArceOS platform IRQ handlers acknowledge the current IRQ // internally. The raw vector argument is ignored by current GIC-backed // platforms, so keep the ack/EOI ownership inside the platform handler. - arceos::handle_host_irq(0) + ax_std::os::arceos::modules::ax_hal::irq::handle_irq(0).then_some(0) } pub(crate) fn fetch_irq() -> usize { diff --git a/virtualization/axvm/src/arch/aarch64/images.rs b/virtualization/axvm/src/arch/aarch64/images.rs new file mode 100644 index 0000000000..f2f256817f --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/images.rs @@ -0,0 +1,33 @@ +//! Public AArch64 image-loader facade preserving the DTB constructor contract. + +use ax_errno::AxResult; +use axvmconfig::AxVMCrateConfig; + +use crate::{ + AxVMRef, VMMemoryRegion, + boot::{BootImageProvider, fdt::GuestDtbImage, images::ImageLoaderCore}, +}; + +pub struct ImageLoader<'a>(ImageLoaderCore<'a>); + +impl<'a> ImageLoader<'a> { + pub fn new( + main_memory: VMMemoryRegion, + config: AxVMCrateConfig, + vm: AxVMRef, + provider: &'a dyn BootImageProvider, + guest_dtb: Option, + ) -> Self { + Self(ImageLoaderCore::new( + main_memory, + config, + vm, + provider, + guest_dtb, + )) + } + + pub fn load(&mut self) -> AxResult { + self.0.load() + } +} diff --git a/virtualization/axvm/src/arch/aarch64/ipi.rs b/virtualization/axvm/src/arch/aarch64/ipi.rs new file mode 100644 index 0000000000..09f99fcc04 --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/ipi.rs @@ -0,0 +1,90 @@ +//! AArch64 guest IPI target resolution and injection. + +use ax_errno::AxResult; + +use super::Aarch64DeferredRunWork; +use crate::architecture::BoundVcpuExit; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct SendIpiExit { + pub(crate) target_cpu: u64, + pub(crate) target_cpu_aux: u64, + pub(crate) send_to_all: bool, + pub(crate) send_to_self: bool, + pub(crate) vector: u64, +} + +pub(crate) fn handle( + vm: &crate::AxVMRef, + vcpu_id: usize, + exit: SendIpiExit, +) -> AxResult> { + let vm_id = vm.id(); + debug!( + "VM[{vm_id}] run VCpu[{vcpu_id}] SendIPI, target_cpu={:#x}, target_cpu_aux={:#x}, \ + vector={}", + exit.target_cpu, exit.target_cpu_aux, exit.vector + ); + let targets = ipi_targets(vm, vcpu_id, exit); + if targets.is_empty() { + warn!( + "VM[{vm_id}] SendIPI has no target: target_cpu={:#x}, target_cpu_aux={:#x}", + exit.target_cpu, exit.target_cpu_aux + ); + return Ok(BoundVcpuExit::Complete( + crate::architecture::VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }, + )); + } + + if targets.get(vcpu_id) { + crate::inject_current_vcpu_interrupt(exit.vector as _) + .expect("failed to inject self IPI into current vCPU"); + } + let mut remote_targets = targets; + remote_targets.set(vcpu_id, false); + if !remote_targets.is_empty() + && let Err(err) = vm.inject_interrupt_to_vcpu(remote_targets, exit.vector as _) + { + warn!( + "Failed to inject interrupt {} to VM[{vm_id}] targets {remote_targets:?}: {err:?}", + exit.vector + ); + } + Ok(BoundVcpuExit::Complete( + crate::architecture::VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }, + )) +} + +fn ipi_targets( + vm: &crate::AxVMRef, + current_vcpu_id: usize, + exit: SendIpiExit, +) -> crate::CpuMask<64> { + let mut targets = crate::CpuMask::new(); + if exit.send_to_all { + for vcpu in vm.vcpu_list() { + if vcpu.id() != current_vcpu_id { + targets.set(vcpu.id(), true); + } + } + } else if exit.send_to_self { + targets.set(current_vcpu_id, true); + } else { + for (vcpu_id, _, phys_id) in vm.get_vcpu_affinities_pcpu_ids() { + let affinity = phys_id as u64; + let aff0 = affinity & 0xff; + let aff123 = affinity & !0xff; + if aff123 == exit.target_cpu && aff0 < 16 && (exit.target_cpu_aux & (1u64 << aff0)) != 0 + { + targets.set(vcpu_id, true); + } + } + } + targets +} diff --git a/virtualization/axvm/src/arch/aarch64/mod.rs b/virtualization/axvm/src/arch/aarch64/mod.rs index 871ad43b5d..962c4f5d0e 100644 --- a/virtualization/axvm/src/arch/aarch64/mod.rs +++ b/virtualization/axvm/src/arch/aarch64/mod.rs @@ -21,14 +21,29 @@ use axvm_types::{ }; use super::{ - ArchOps, BoundVcpuExit, CpuUpExit, HypercallExit, MmioReadExit, MmioWriteExit, SendIpiExit, - SysRegReadExit, SysRegWriteExit, VcpuCreateContext, VcpuRunAction, VcpuSetupContext, - target_phys_cpu_ids, + ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, + VcpuRunAction, VcpuSetupContext, +}; +use crate::{ + architecture::ops::target_phys_cpu_ids, + host::{HostCpu, HostMemory, HostTime, default_host}, }; -use crate::host::{HostCpu, HostMemory, HostTime, default_host}; +mod capabilities; +#[path = "../../architecture/cpu_up.rs"] +mod cpu_up; mod gic; +mod images; +mod ipi; mod npt; +#[path = "../../architecture/sysreg.rs"] +mod sysreg; + +pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; +use cpu_up::{CpuUpExit, CpuUpOps}; +pub use images::ImageLoader; +use ipi::SendIpiExit; +use sysreg::{SysRegReadExit, SysRegWriteExit}; pub(crate) struct Aarch64Arch; @@ -37,6 +52,8 @@ pub(crate) enum Aarch64DeferredRunWork { ExternalInterrupt { vector: usize }, } +impl CpuUpOps for Aarch64Arch {} + impl ArchOps for Aarch64Arch { type VCpu = AxvmArmVcpu; type PerCpu = AxvmArmPerCpu; @@ -121,52 +138,25 @@ impl ArchOps for Aarch64Arch { _state: &Self::VcpuCreateState, ctx: VcpuCreateContext, ) -> AxResult<::CreateConfig> { + let (_vcpu_id, phys_cpu_id, dtb_addr, _firmware_boot) = ctx.into_parts(); Ok(ArmVcpuCreateConfig { - mpidr_el1: ctx.phys_cpu_id as _, - dtb_addr: ctx.dtb_addr.unwrap_or_default().as_usize(), + mpidr_el1: phys_cpu_id as _, + dtb_addr: dtb_addr.unwrap_or_default().as_usize(), }) } fn build_vcpu_setup_config( ctx: VcpuSetupContext<'_>, ) -> AxResult<::SetupConfig> { - let passthrough = ctx.interrupt_mode == axvm_types::VMInterruptMode::Passthrough; + let (interrupt_mode, _console, _ports, _memory_regions, _firmware_boot) = + ctx.into_parts(); + let passthrough = interrupt_mode == axvm_types::VMInterruptMode::Passthrough; Ok(ArmVcpuSetupConfig { passthrough_interrupt: passthrough, passthrough_timer: passthrough, }) } - fn ipi_targets( - vm: &crate::AxVMRef, - current_vcpu_id: usize, - target_cpu: u64, - target_cpu_aux: u64, - send_to_all: bool, - send_to_self: bool, - ) -> crate::CpuMask<64> { - let mut targets = crate::CpuMask::new(); - if send_to_all { - for vcpu in vm.vcpu_list() { - if vcpu.id() != current_vcpu_id { - targets.set(vcpu.id(), true); - } - } - } else if send_to_self { - targets.set(current_vcpu_id, true); - } else { - for (vcpu_id, _, phys_id) in vm.get_vcpu_affinities_pcpu_ids() { - let affinity = phys_id as u64; - let aff0 = affinity & 0xff; - let aff123 = affinity & !0xff; - if aff123 == target_cpu && aff0 < 16 && (target_cpu_aux & (1u64 << aff0)) != 0 { - targets.set(vcpu_id, true); - } - } - } - targets - } - fn handle_vcpu_exit_bound( vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef, @@ -201,7 +191,7 @@ impl ArchOps for Aarch64Arch { data, }, ), - ArmVmExit::SysRegRead { addr, reg } => super::handle_sys_reg_read( + ArmVmExit::SysRegRead { addr, reg } => sysreg::handle_read( vm, vcpu, SysRegReadExit { @@ -209,7 +199,7 @@ impl ArchOps for Aarch64Arch { reg, }, ), - ArmVmExit::SysRegWrite { addr, value } => super::handle_sys_reg_write( + ArmVmExit::SysRegWrite { addr, value } => sysreg::handle_write( vm, SysRegWriteExit { addr: arm_sys_reg_addr_to_ax(addr), @@ -230,13 +220,16 @@ impl ArchOps for Aarch64Arch { vm.id(), vcpu.id() ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Wait)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: true, + stop_reason: None, + })) } ArmVmExit::CpuUp { target_cpu, entry_point, arg, - } => super::handle_cpu_up::( + } => cpu_up::handle::( vm, vcpu, CpuUpExit { @@ -247,9 +240,10 @@ impl ArchOps for Aarch64Arch { ), ArmVmExit::SystemDown => { warn!("VM[{}] run VCpu[{}] SystemDown", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Stop( - crate::StopReason::SystemDown, - ))) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: Some(crate::StopReason::SystemDown), + })) } ArmVmExit::SendIPI { target_cpu, @@ -257,7 +251,7 @@ impl ArchOps for Aarch64Arch { send_to_all, send_to_self, vector, - } => super::handle_send_ipi::( + } => ipi::handle( vm, vcpu.id(), SendIpiExit { @@ -268,7 +262,10 @@ impl ArchOps for Aarch64Arch { vector, }, ), - ArmVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)), + ArmVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })), _ => ax_err!(Unsupported, "unsupported AArch64 VM exit"), } } @@ -283,7 +280,10 @@ impl ArchOps for Aarch64Arch { Self::after_external_interrupt(vm, vcpu, vector); } } - Ok(VcpuRunAction::Yield) + Ok(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }) } } diff --git a/virtualization/axvm/src/arch/aarch64/npt.rs b/virtualization/axvm/src/arch/aarch64/npt.rs index f480db919b..d0a05a445b 100644 --- a/virtualization/axvm/src/arch/aarch64/npt.rs +++ b/virtualization/axvm/src/arch/aarch64/npt.rs @@ -231,7 +231,7 @@ impl ptg::TableMeta for A64HVPagingMetaDataL4 { } pub(crate) type NestedPageTable = - crate::arch::npt::LeveledPageTable; + crate::npt::LeveledPageTable; fn config_to_flags(config: ptg::PteConfig) -> MappingFlags { let mut flags = MappingFlags::empty(); diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/guest_firmware_dtb.rs b/virtualization/axvm/src/arch/loongarch64/boot/fdt/guest_firmware_dtb.rs similarity index 99% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/guest_firmware_dtb.rs rename to virtualization/axvm/src/arch/loongarch64/boot/fdt/guest_firmware_dtb.rs index 5d7c3ca36e..dc8ecab222 100644 --- a/virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/guest_firmware_dtb.rs +++ b/virtualization/axvm/src/arch/loongarch64/boot/fdt/guest_firmware_dtb.rs @@ -4,7 +4,7 @@ use ax_errno::{AxResult, ax_err_type}; use fdt_edit::{Fdt, Node, NodeId}; use super::property::{prop_null, prop_string, prop_string_list, prop_u32, prop_u32_array}; -use crate::boot::guest_platform::loongarch64::GuestPlatform; +use crate::arch::guest_platform::GuestPlatform; const PHANDLE_CPU0: u32 = 0x8000; const PHANDLE_CPUIC: u32 = 0x8001; diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/boot/fdt/mod.rs similarity index 100% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/mod.rs rename to virtualization/axvm/src/arch/loongarch64/boot/fdt/mod.rs diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/property.rs b/virtualization/axvm/src/arch/loongarch64/boot/fdt/property.rs similarity index 100% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/loongarch64/property.rs rename to virtualization/axvm/src/arch/loongarch64/boot/fdt/property.rs diff --git a/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/mod.rs b/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/mod.rs deleted file mode 100644 index 80c43742c4..0000000000 --- a/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[cfg(target_arch = "loongarch64")] -pub mod loongarch64; diff --git a/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/boot/mod.rs similarity index 55% rename from virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/mod.rs rename to virtualization/axvm/src/arch/loongarch64/boot/mod.rs index 4b17fba3e8..250234803e 100644 --- a/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/mod.rs +++ b/virtualization/axvm/src/arch/loongarch64/boot/mod.rs @@ -1,22 +1,53 @@ +mod fdt; mod probe; mod resources; -use alloc::{boxed::Box, vec::Vec}; +use alloc::{boxed::Box, format, vec::Vec}; use ax_errno::{AxResult, ax_err_type}; use axdevice::{ FwCfgInterruptConfig, FwCfgPciConfig, FwCfgPlatformConfig, FwCfgRamRegion, FwCfgSerialConfig, }; -use axvmconfig::{AxVMCrateConfig, EmulatedDeviceType}; +use axvmconfig::{AxVMCrateConfig, EmulatedDeviceType, VMBootProtocol}; pub use resources::{ LoongArchGuestIrqRoute, get_guest_irq_routes, prepare_uefi_fdt_config, prepare_uefi_runtime_config, }; -use crate::{AxVMRef, GuestPhysAddr, boot::images::load_vm_image_from_memory}; +use crate::{ + AxVMRef, GuestPhysAddr, + architecture::BootImagePlatform, + boot::{ + BootImageProvider, StaticVmImage, + images::{ImageLoaderCore, load_vm_image_from_memory}, + }, +}; pub const UEFI_FIRMWARE_FDT_BASE: usize = 0x0010_0000; +pub struct ImageLoader<'a>(ImageLoaderCore<'a>); + +impl<'a> ImageLoader<'a> { + pub fn new( + main_memory: crate::VMMemoryRegion, + config: AxVMCrateConfig, + vm: AxVMRef, + provider: &'a dyn BootImageProvider, + ) -> Self { + Self(ImageLoaderCore::new( + main_memory, + config, + vm, + provider, + None, + )) + } + + pub fn load(&mut self) -> AxResult { + self.0.load() + } +} + pub fn init() { resources::init(); } @@ -145,7 +176,7 @@ impl GuestPlatform { pub fn load_firmware_fdt(vm: &AxVMRef, config: &AxVMCrateConfig) -> AxResult { let platform = GuestPlatform::discover(vm, config); - let fdt = crate::boot::fdt::loongarch64::guest_firmware_dtb::build(&platform)?; + let fdt = fdt::guest_firmware_dtb::build(&platform)?; debug!( "VM[{}] loading LoongArch UEFI firmware FDT: {} bytes at {:#x}", config.base.id, @@ -187,6 +218,128 @@ pub fn emulated_fw_cfg(config: &AxVMCrateConfig) -> AxResult<&axvmconfig::Emulat .ok_or_else(|| ax_err_type!(NotFound, "LoongArch UEFI boot requires a fw_cfg device")) } +impl BootImagePlatform for super::LoongArch64Arch { + fn load_images_from_memory( + loader: &mut ImageLoaderCore<'_>, + images: StaticVmImage, + ) -> AxResult { + ensure_uefi_boot(loader)?; + load_uefi_firmware_dtb(loader)?; + add_uefi_fw_cfg(loader, images.kernel, images.ramdisk)?; + let firmware = images + .bios + .or_else(|| provider_firmware_image(loader)) + .ok_or_else(|| { + ax_err_type!( + NotFound, + "LoongArch UEFI boot requires a build-time firmware image" + ) + })?; + load_uefi_firmware_image(loader, firmware) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + fn load_images_from_filesystem(loader: &mut ImageLoaderCore<'_>) -> AxResult { + ensure_uefi_boot(loader)?; + load_uefi_firmware_dtb(loader)?; + + let kernel = crate::boot::images::fs::read_full_image( + &loader.config.kernel.kernel_path, + loader.provider, + )?; + let kernel: &'static [u8] = Box::leak(kernel.into_boxed_slice()); + let ramdisk = if let Some(path) = &loader.config.kernel.ramdisk_path { + let ramdisk = crate::boot::images::fs::read_full_image(path, loader.provider)?; + Some(Box::leak(ramdisk.into_boxed_slice()) as &'static [u8]) + } else { + None + }; + add_uefi_fw_cfg(loader, kernel, ramdisk)?; + + let firmware = provider_firmware_image(loader).ok_or_else(|| { + ax_err_type!( + NotFound, + "LoongArch UEFI boot requires a build-time firmware image" + ) + })?; + load_uefi_firmware_image(loader, firmware) + } +} + +fn ensure_uefi_boot(loader: &ImageLoaderCore<'_>) -> AxResult { + if loader.config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { + Ok(()) + } else { + ax_errno::ax_err!(Unsupported, "LoongArch guests require UEFI boot") + } +} + +fn load_uefi_firmware_dtb(loader: &ImageLoaderCore<'_>) -> AxResult { + prepare_uefi_runtime_config(&loader.vm, &loader.config); + load_firmware_fdt(&loader.vm, &loader.config) +} + +fn add_uefi_fw_cfg( + loader: &ImageLoaderCore<'_>, + kernel: &'static [u8], + ramdisk: Option<&'static [u8]>, +) -> AxResult { + let fw_cfg = emulated_fw_cfg(&loader.config)?; + loader.vm.add_fw_cfg_device(crate::FwCfgDeviceConfig { + base: GuestPhysAddr::from(fw_cfg.base_gpa), + size: fw_cfg.length, + kernel, + initrd: ramdisk, + cmdline: loader.config.kernel.cmdline.clone(), + cpu_num: loader.config.base.cpu_num as u16, + platform: fw_cfg_platform_config(&loader.vm, &loader.config), + }) +} + +fn provider_firmware_image(loader: &ImageLoaderCore<'_>) -> Option<&'static [u8]> { + loader + .provider + .static_firmware_images() + .iter() + .find(|image| image.id == loader.config.base.id) + .and_then(|image| image.bios) +} + +fn load_uefi_firmware_image(loader: &ImageLoaderCore<'_>, firmware: &[u8]) -> AxResult { + let load_gpa = loader + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "LoongArch UEFI firmware load addr is missed"))?; + let flash_len = loader + .config + .kernel + .memory_regions + .iter() + .find(|region| region.gpa == load_gpa.as_usize()) + .map_or(firmware.len(), |region| region.size); + fill_vm_region(load_gpa, flash_len, 0xff, loader.vm.clone())?; + load_vm_image_from_memory(firmware, load_gpa, loader.vm.clone()) +} + +fn fill_vm_region(load_addr: GuestPhysAddr, size: usize, byte: u8, vm: AxVMRef) -> AxResult { + let regions = vm.get_image_load_region(load_addr, size)?; + let mut filled_size = 0; + for region in regions { + // SAFETY: AxVM returned this writable guest-memory region and the fill + // is bounded by its length. + unsafe { core::ptr::write_bytes(region.as_mut_ptr(), byte, region.len()) }; + crate::clean_dcache_range((region.as_ptr() as usize).into(), region.len()); + filled_size += region.len(); + } + if filled_size == size { + Ok(()) + } else { + ax_errno::ax_err!( + InvalidData, + format!("VM memory was only partially filled: {filled_size}/{size} bytes") + ) + } +} + fn ram_regions(vm: &AxVMRef) -> Vec { let mut regions = vm .memory_regions() diff --git a/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/probe.rs b/virtualization/axvm/src/arch/loongarch64/boot/probe.rs similarity index 100% rename from virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/probe.rs rename to virtualization/axvm/src/arch/loongarch64/boot/probe.rs diff --git a/virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/resources.rs b/virtualization/axvm/src/arch/loongarch64/boot/resources.rs similarity index 100% rename from virtualization/axvm/src/arch/loongarch64/boot/guest_platform/loongarch64/resources.rs rename to virtualization/axvm/src/arch/loongarch64/boot/resources.rs diff --git a/virtualization/axvm/src/arch/loongarch64/capabilities.rs b/virtualization/axvm/src/arch/loongarch64/capabilities.rs new file mode 100644 index 0000000000..c712d6d8f1 --- /dev/null +++ b/virtualization/axvm/src/arch/loongarch64/capabilities.rs @@ -0,0 +1,26 @@ +//! LoongArch64 implementations of AxVM platform capability hooks. + +use super::LoongArch64Arch; +use crate::architecture::{AddressSpacePlatform, DevicePlatform, HostTimePlatform}; + +impl DevicePlatform for LoongArch64Arch {} + +impl AddressSpacePlatform for LoongArch64Arch {} + +impl HostTimePlatform for LoongArch64Arch { + fn set_oneshot_timer(_deadline_ns: u64) {} + + fn register_timer_callback() { + ax_std::os::arceos::modules::ax_task::register_timer_callback(|_| { + crate::check_timer_events(); + }); + } +} + +pub fn host_fdt_bootarg() -> usize { + ax_std::os::arceos::modules::ax_hal::dtb::get_bootarg() +} + +pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + ax_std::os::arceos::modules::ax_hal::mem::phys_to_virt(paddr) +} diff --git a/virtualization/axvm/src/arch/loongarch64/idle.rs b/virtualization/axvm/src/arch/loongarch64/idle.rs new file mode 100644 index 0000000000..4b97b55d50 --- /dev/null +++ b/virtualization/axvm/src/arch/loongarch64/idle.rs @@ -0,0 +1,26 @@ +//! LoongArch guest idle handling while host interrupts remain available. + +use super::AxvmLoongArchVcpu; + +pub(crate) fn wait(vcpu: &crate::vm::AxVCpuRef) { + crate::check_timer_events(); + if vcpu.get_arch_vcpu().has_enabled_pending_interrupt() { + trace!( + "VM[{}] VCpu[{}] skips idle wait because guest has enabled pending interrupt", + vcpu.vm_id(), + vcpu.id() + ); + return; + } + let idle_timeout = vcpu.get_arch_vcpu().idle_wait_timeout(); + trace!( + "VM[{}] VCpu[{}] host idle wait for {idle_timeout:?}", + vcpu.vm_id(), + vcpu.id() + ); + ax_std::os::arceos::modules::ax_hal::asm::set_timer_irq_enabled(true); + ax_std::os::arceos::modules::ax_hal::asm::enable_irqs(); + ax_std::os::arceos::modules::ax_hal::time::busy_wait(idle_timeout); + ax_std::os::arceos::modules::ax_hal::asm::disable_irqs(); + ax_std::os::arceos::modules::ax_hal::asm::set_timer_irq_enabled(false); +} diff --git a/virtualization/axvm/src/arch/loongarch64/irq.rs b/virtualization/axvm/src/arch/loongarch64/irq.rs index 2040e974e8..3a205f3923 100644 --- a/virtualization/axvm/src/arch/loongarch64/irq.rs +++ b/virtualization/axvm/src/arch/loongarch64/irq.rs @@ -5,7 +5,7 @@ const EIOINTC_IRQ: usize = 3; /// Register the platform IRQ injector for LoongArch dynamic hypervisor builds. pub(crate) fn register_platform_irq_injector() { ax_plat::irq::loongarch64_hv::register_virtual_irq_injector(inject_platform_irq); - crate::host::arceos::set_irq_enabled(EIOINTC_IRQ, true); + set_irq_enabled(EIOINTC_IRQ, true); } /// Route a host physical IRQ to a LoongArch guest interrupt vector. @@ -28,6 +28,31 @@ pub fn unregister_guest_irq_routes(vm_id: usize) { ax_plat::irq::loongarch64_hv::unregister_guest_irq_routes(vm_id); } +fn set_irq_enabled(raw_irq: usize, enabled: bool) { + use ax_std::os::arceos::modules::ax_hal::irq::{self, IrqSource}; + + let gsi = match u32::try_from(raw_irq) { + Ok(gsi) => gsi, + Err(_) => { + warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: out of GSI range"); + return; + } + }; + let irq = match irq::resolve_irq_source(IrqSource::AcpiGsi(gsi)) { + Ok(irq) => irq, + Err(err) => { + warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: {err:?}"); + return; + } + }; + if let Err(err) = irq::set_enable(irq, enabled) { + warn!( + "failed to set LoongArch passthrough IRQ {raw_irq} ({irq:?}) enabled={enabled}: \ + {err:?}" + ); + } +} + fn inject_platform_irq(vm_id: usize, vcpu_id: usize, vector: usize, physical_irq: usize) { if let Err(err) = crate::runtime::vcpus::queue_external_interrupt(vm_id, vcpu_id, vector, physical_irq) diff --git a/virtualization/axvm/src/arch/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/mod.rs index 14e804b9b1..87aa9d4d1f 100644 --- a/virtualization/axvm/src/arch/loongarch64/mod.rs +++ b/virtualization/axvm/src/arch/loongarch64/mod.rs @@ -21,9 +21,14 @@ use super::{ }; use crate::host::{HostMemory, HostTime, default_host}; +pub(crate) mod boot; +mod capabilities; +mod idle; pub(crate) mod irq; mod npt; +pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; + pub(crate) struct LoongArch64Arch; #[derive(Clone, Copy, Debug)] @@ -58,12 +63,13 @@ impl ArchOps for LoongArch64Arch { state: &Self::VcpuCreateState, ctx: VcpuCreateContext, ) -> AxResult<::CreateConfig> { + let (vcpu_id, _phys_cpu_id, dtb_addr, firmware_boot) = ctx.into_parts(); Ok(LoongArchVCpuCreateConfig { - cpu_id: ctx.vcpu_id, - dtb_addr: ctx.dtb_addr.unwrap_or_default().as_usize(), + cpu_id: vcpu_id, + dtb_addr: dtb_addr.unwrap_or_default().as_usize(), boot_args: [0; 3], boot_stack_top: 0, - firmware_boot: ctx.firmware_boot, + firmware_boot, iocsr_state: state.clone(), }) } @@ -71,13 +77,15 @@ impl ArchOps for LoongArch64Arch { fn build_vcpu_setup_config( ctx: VcpuSetupContext<'_>, ) -> AxResult<::SetupConfig> { - let passthrough = ctx.interrupt_mode == axvm_types::VMInterruptMode::Passthrough; + let (interrupt_mode, _console, _ports, _memory_regions, firmware_boot) = + ctx.into_parts(); + let passthrough = interrupt_mode == axvm_types::VMInterruptMode::Passthrough; Ok(LoongArchVCpuSetupConfig { passthrough_interrupt: passthrough, passthrough_timer: passthrough, boot_args: [0; 3], boot_stack_top: 0, - firmware_boot: ctx.firmware_boot, + firmware_boot, }) } @@ -147,29 +155,6 @@ impl ArchOps for LoongArch64Arch { drain_loongarch_pch_pic_events(vm); } - fn handle_idle(_vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { - crate::check_timer_events(); - if vcpu.get_arch_vcpu().has_enabled_pending_interrupt() { - trace!( - "VM[{}] VCpu[{}] skips idle wait because guest has enabled pending interrupt", - vcpu.vm_id(), - vcpu.id() - ); - return; - } - let idle_timeout = vcpu.get_arch_vcpu().idle_wait_timeout(); - trace!( - "VM[{}] VCpu[{}] host idle wait for {idle_timeout:?}", - vcpu.vm_id(), - vcpu.id() - ); - ax_std::os::arceos::modules::ax_hal::asm::set_timer_irq_enabled(true); - ax_std::os::arceos::modules::ax_hal::asm::enable_irqs(); - ax_std::os::arceos::modules::ax_hal::time::busy_wait(idle_timeout); - ax_std::os::arceos::modules::ax_hal::asm::disable_irqs(); - ax_std::os::arceos::modules::ax_hal::asm::set_timer_irq_enabled(false); - } - fn handle_vcpu_exit_bound( vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef, @@ -221,9 +206,15 @@ impl ArchOps for LoongArch64Arch { } LoongArchVmExit::Halt => { debug!("VM[{}] run VCpu[{}] Halt", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(Self::handle_halt())) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: true, + stop_reason: None, + })) } - LoongArchVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)), + LoongArchVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })), _ => Err(AxError::Unsupported), } } @@ -237,9 +228,12 @@ impl ArchOps for LoongArch64Arch { LoongArchDeferredRunWork::ExternalInterrupt { vector } => { Self::after_external_interrupt(vm, vcpu, vector); } - LoongArchDeferredRunWork::Idle => Self::handle_idle(vm, vcpu), + LoongArchDeferredRunWork::Idle => idle::wait(vcpu), } - Ok(VcpuRunAction::Yield) + Ok(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }) } fn clean_dcache_range(addr: VirtAddr, size: usize) { @@ -265,7 +259,10 @@ fn handle_loongarch_nested_page_fault( vcpu.id(), ax_addr.as_usize() ); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); + return Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })); }; return LoongArch64Arch::handle_vcpu_exit_bound(vm, vcpu, decoded); } @@ -281,7 +278,10 @@ fn handle_loongarch_nested_page_fault( ax_addr.as_usize(), ax_flags ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } } @@ -348,7 +348,7 @@ impl LoongArchHostOps for AxvmLoongArchHostOps { } fn cancel_timer(token: usize) { - default_host().cancel_timer(token); + crate::timer::cancel_timer(token); } fn inject_interrupt(vm_id: usize, vcpu_id: usize, vector: usize) { diff --git a/virtualization/axvm/src/arch/loongarch64/npt.rs b/virtualization/axvm/src/arch/loongarch64/npt.rs index 6eaa824ff8..876d1a865a 100644 --- a/virtualization/axvm/src/arch/loongarch64/npt.rs +++ b/virtualization/axvm/src/arch/loongarch64/npt.rs @@ -197,12 +197,8 @@ impl ptg::TableMeta for LoongArchPagingMetaDataL4 { } } -pub(crate) type NestedPageTable = crate::arch::npt::LeveledPageTable< - LoongArchPagingMetaDataL3, - LoongArchPagingMetaDataL4, - H, - true, ->; +pub(crate) type NestedPageTable = + crate::npt::LeveledPageTable; fn config_to_flags(config: ptg::PteConfig) -> MappingFlags { let mut flags = MappingFlags::empty(); diff --git a/virtualization/axvm/src/arch/mod.rs b/virtualization/axvm/src/arch/mod.rs index 8de28e7f10..fdab0d56b5 100644 --- a/virtualization/axvm/src/arch/mod.rs +++ b/virtualization/axvm/src/arch/mod.rs @@ -1,79 +1,73 @@ -//! Architecture component glue owned by AxVM. +//! Target architecture selection and stable internal dispatch. -use alloc::{format, vec::Vec}; +use ax_errno::AxResult; +use ax_memory_addr::PhysAddr; +use axvm_types::NestedPagingConfig; -use ax_errno::{AxResult, ax_err}; -use ax_memory_addr::{PhysAddr, VirtAddr}; -use axaddrspace::NestedPageTableOps; -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -use axvm_types::SysRegAddr; -use axvm_types::{ - AccessWidth, GuestPhysAddr, NestedPagingConfig, PassThroughPortConfig, VMInterruptMode, - VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState, +pub(crate) use crate::architecture::*; +use crate::architecture::{ + AddressSpacePlatform, BootImagePlatform, DevicePlatform, HostTimePlatform, }; -#[cfg(target_arch = "x86_64")] -use axvm_types::{MappingFlags, Port}; - -#[cfg(target_arch = "aarch64")] -use crate::CpuMask; -use crate::StopReason; #[cfg(target_arch = "aarch64")] mod aarch64; -#[path = "x86_64/host.rs"] -pub mod arceos; -#[path = "aarch64/boot/fdt/mod.rs"] -pub mod fdt; -#[path = "loongarch64/boot/guest_platform/mod.rs"] -pub mod guest_platform; -#[path = "x86_64/boot/images/mod.rs"] -pub mod images; #[cfg(target_arch = "loongarch64")] mod loongarch64; -mod npt; #[cfg(target_arch = "riscv64")] mod riscv64; #[cfg(target_arch = "x86_64")] mod x86_64; +#[cfg(target_arch = "aarch64")] +pub(crate) use aarch64::Aarch64Arch as CurrentArch; +#[cfg(target_arch = "aarch64")] +pub use aarch64::ImageLoader; +#[cfg(target_arch = "loongarch64")] +pub(crate) use loongarch64::LoongArch64Arch as CurrentArch; +#[cfg(target_arch = "loongarch64")] +pub(crate) use loongarch64::boot as guest_platform; +#[cfg(target_arch = "loongarch64")] +pub use loongarch64::boot::ImageLoader; +#[cfg(not(target_arch = "loongarch64"))] +pub(crate) mod guest_platform { + #[doc(hidden)] + pub const SUPPORTED: bool = false; +} +#[cfg(target_arch = "riscv64")] +pub use riscv64::ImageLoader; +#[cfg(target_arch = "riscv64")] +pub(crate) use riscv64::Riscv64Arch as CurrentArch; +#[cfg(target_arch = "x86_64")] +pub(crate) use x86_64::X86_64Arch as CurrentArch; +#[cfg(target_arch = "x86_64")] +pub use x86_64::boot::ImageLoader; + /// Architecture-specific public compatibility exports. -/// -/// The parent crate re-exports this module so legacy AxVM API paths remain -/// stable while target selection stays contained within `arch`. pub mod platform { - #[cfg(all( - any(target_arch = "x86_64", target_arch = "loongarch64"), - any(feature = "fs", feature = "host-fs") - ))] - pub use super::arceos::shutdown_host_filesystems; - #[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" - ))] - pub use super::arceos::{host_fdt_bootarg, phys_to_virt as host_phys_to_virt}; + #[cfg(target_arch = "aarch64")] + pub use super::aarch64::{host_fdt_bootarg, host_phys_to_virt}; #[cfg(target_arch = "loongarch64")] pub use super::loongarch64::irq::{ register_guest_irq_route as register_loongarch_guest_irq_route, unregister_guest_irq_routes as unregister_loongarch_guest_irq_routes, }; + #[cfg(target_arch = "loongarch64")] + pub use super::loongarch64::{host_fdt_bootarg, host_phys_to_virt}; + #[cfg(target_arch = "riscv64")] + pub use super::riscv64::{host_fdt_bootarg, host_phys_to_virt}; #[cfg(target_arch = "x86_64")] pub use super::x86_64::irq::{ register_ioapic_irq_forwarding_activator as register_x86_ioapic_irq_forwarding_activator, register_ioapic_irq_forwarding_route as register_x86_ioapic_irq_forwarding_route, register_ioapic_irq_forwarding_route_with_trigger as register_x86_ioapic_irq_forwarding_route_with_trigger, }; + #[cfg(all( + any(target_arch = "x86_64", target_arch = "loongarch64"), + any(feature = "fs", feature = "host-fs") + ))] + pub use crate::host::arceos::shutdown_host_filesystems; } -#[cfg(target_arch = "aarch64")] -pub(crate) type CurrentArch = aarch64::Aarch64Arch; -#[cfg(target_arch = "loongarch64")] -pub(crate) type CurrentArch = loongarch64::LoongArch64Arch; -#[cfg(target_arch = "riscv64")] -pub(crate) type CurrentArch = riscv64::Riscv64Arch; -#[cfg(target_arch = "x86_64")] -pub(crate) type CurrentArch = x86_64::X86_64Arch; - pub(crate) type ArchVCpu = ::VCpu; pub(crate) type ArchPerCpu = ::PerCpu; pub(crate) type ArchNestedPageTable = ::NestedPageTable; @@ -96,785 +90,59 @@ pub(crate) fn nested_paging_config( CurrentArch::nested_paging_config(root_paddr, levels, vcpu_mappings) } -/// Creates the VM interrupt fabric and registers architecture-owned factories. -#[cfg(target_arch = "riscv64")] pub(crate) fn configure_interrupt_fabric( factories: &mut axdevice::DeviceFactoryRegistry, - mode: VMInterruptMode, + mode: axvm_types::VMInterruptMode, configs: &[axvm_types::EmulatedDeviceConfig], ) -> AxResult { - riscv64::irq::configure(factories, mode, configs) -} - -/// Creates the VM interrupt fabric when no architecture-owned backend is needed. -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "x86_64" -))] -pub(crate) fn configure_interrupt_fabric( - _factories: &mut axdevice::DeviceFactoryRegistry, - mode: VMInterruptMode, - _configs: &[axvm_types::EmulatedDeviceConfig], -) -> AxResult { - Ok(crate::InterruptFabric::new(mode)) -} - -/// Adds architecture-owned guest physical ranges to the address-layout planner. -#[cfg(all(target_arch = "x86_64", feature = "vmx"))] -pub(crate) fn append_arch_owned_regions(regions: &mut Vec) { - regions.push(crate::layout::GuestOwnedRegion::new( - x86_vcpu::X86_APIC_ACCESS_GPA, - ax_memory_addr::PAGE_SIZE_4K, - crate::layout::VmRegionKind::Reserved, - )); -} - -/// Adds no additional guest physical ranges for this architecture configuration. -#[cfg(not(all(target_arch = "x86_64", feature = "vmx")))] -pub(crate) fn append_arch_owned_regions(_regions: &mut Vec) {} - -/// Installs architecture-owned stage-2 mappings after the generic layout. -#[cfg(all(target_arch = "x86_64", feature = "vmx"))] -pub(crate) fn map_arch_address_space( - address_space: &mut axaddrspace::AddrSpace, -) -> AxResult { - address_space.map_linear( - axvm_types::GuestPhysAddr::from(x86_vcpu::X86_APIC_ACCESS_GPA), - x86_64::x86_apic_access_page_addr(), - ax_memory_addr::PAGE_SIZE_4K, - axvm_types::MappingFlags::DEVICE - | axvm_types::MappingFlags::READ - | axvm_types::MappingFlags::WRITE, - ) -} - -/// Installs no additional stage-2 mappings for this architecture configuration. -#[cfg(not(all(target_arch = "x86_64", feature = "vmx")))] -pub(crate) fn map_arch_address_space( - _address_space: &mut axaddrspace::AddrSpace, -) -> AxResult { - Ok(()) + CurrentArch::configure_interrupt_fabric(factories, mode, configs) } -/// Registers the host timer callback required by the active architecture. -#[cfg(target_arch = "loongarch64")] -pub(crate) fn register_timer_callback() { - ax_std::os::arceos::modules::ax_task::register_timer_callback(|_| crate::check_timer_events()); -} - -/// Registers no timer callback when the host timer interrupts directly. -#[cfg(any( - target_arch = "aarch64", - target_arch = "riscv64", - target_arch = "x86_64" -))] -pub(crate) fn register_timer_callback() {} - -/// Registers devices and passthrough resources owned by the active architecture. -#[cfg(target_arch = "aarch64")] pub(crate) fn register_arch_devices( vm: &crate::AxVM, config: &crate::config::AxVMConfig, devices: &mut axdevice::AxVmDevices, ) -> AxResult { - use alloc::sync::Arc; - - use ax_errno::ax_err_type; - use axdevice_base::DeviceRegistry as _; - - if config.interrupt_mode() == VMInterruptMode::Passthrough { - let cpu_id = vm.id() - 1; // FIXME: get the real CPU id. - let mut gicd_found = false; - for device in devices.devices() { - if let Some(gicd) = device.as_any().downcast_ref::() { - for spi in config.pass_through_spis() { - gicd.assign_irq(*spi + 32, cpu_id, (0, 0, 0, cpu_id as _)); - } - gicd_found = true; - break; - } - } - if !gicd_found { - warn!("Failed to assign SPIs: No VGicD found in device list"); - } - } else { - for device in axdevice::create_vtimer_devices() { - devices - .register(Arc::from(device) as Arc) - .map_err(|err| ax_err_type!(InvalidInput, format!("register vtimer: {err:?}")))?; - } - } - Ok(()) + CurrentArch::register_devices(vm, config, devices) } -/// Registers devices and passthrough resources owned by the active architecture. -#[cfg(target_arch = "x86_64")] -pub(crate) fn register_arch_devices( - _vm: &crate::AxVM, - config: &crate::config::AxVMConfig, - devices: &mut axdevice::AxVmDevices, -) -> AxResult { - use alloc::{format, sync::Arc}; - - use ax_errno::ax_err_type; - use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; - - for port in config.pass_through_ports() { - let passthrough = Arc::new(x86_64::port::HostPortPassthrough::new( - port.base, - port.length, - )?); - let range = passthrough.address_range(); - debug!( - "PT port region: [{:#x}~{:#x}]", - range.start.number(), - range.end.number(), - ); - devices - .register(PortDeviceAdapter::from_arc(passthrough)) - .map_err(|err| ax_err_type!(InvalidInput, format!("register PT port: {err:?}")))?; - } - for config in config.emu_devices() { - x86_64::register_arch_device(config, devices)?; - } - Ok(()) +pub(crate) fn append_arch_owned_regions( + regions: &mut alloc::vec::Vec, +) { + CurrentArch::append_owned_regions(regions); } -/// Registers devices and passthrough resources owned by the active architecture. -#[cfg(any(target_arch = "loongarch64", target_arch = "riscv64"))] -pub(crate) fn register_arch_devices( - _vm: &crate::AxVM, - _config: &crate::config::AxVMConfig, - _devices: &mut axdevice::AxVmDevices, +pub(crate) fn map_arch_address_space( + address_space: &mut axaddrspace::AddrSpace, ) -> AxResult { - Ok(()) -} - -/// Runtime scheduler action selected after an architecture-local vCPU exit. -#[derive(Debug)] -pub(crate) enum VcpuRunAction { - /// Return to the runtime loop without blocking. - Yield, - /// Block the current vCPU task on the VM runtime wait queue. - Wait, - /// Request VM stop with the provided reason. - Stop(StopReason), -} - -/// Result of handling one exit while the vCPU is still bound to the host CPU. -#[derive(Debug)] -pub(crate) enum BoundVcpuExit { - /// The exit was handled completely; re-enter the guest in the current run slice. - Continue, - /// The run slice is complete and can return this scheduler action after unbind. - Complete(VcpuRunAction), - /// Finish architecture-local work after unbinding the vCPU. - Defer(D), -} - -#[cfg(target_arch = "x86_64")] -#[derive(Clone, Copy, Debug)] -pub(crate) enum LegacyDeferredRunWork { - ExternalInterrupt { vector: usize }, - PreemptionTimer, - InterruptEnd { vector: Option }, -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct MmioReadExit { - pub(crate) addr: GuestPhysAddr, - pub(crate) width: AccessWidth, - pub(crate) reg: usize, - pub(crate) reg_width: AccessWidth, - pub(crate) signed_ext: bool, -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct MmioWriteExit { - pub(crate) addr: GuestPhysAddr, - pub(crate) width: AccessWidth, - pub(crate) data: u64, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(target_arch = "x86_64")] -pub(crate) struct IoReadExit { - pub(crate) port: Port, - pub(crate) width: AccessWidth, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(target_arch = "x86_64")] -pub(crate) struct IoWriteExit { - pub(crate) port: Port, - pub(crate) width: AccessWidth, - pub(crate) data: u64, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub(crate) struct SysRegReadExit { - pub(crate) addr: SysRegAddr, - pub(crate) reg: usize, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub(crate) struct SysRegWriteExit { - pub(crate) addr: SysRegAddr, - pub(crate) value: u64, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(target_arch = "x86_64")] -pub(crate) struct NestedPageFaultExit { - pub(crate) addr: GuestPhysAddr, - pub(crate) access_flags: MappingFlags, -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct HypercallExit { - pub(crate) nr: u64, - pub(crate) args: [u64; 6], -} - -#[derive(Clone, Copy, Debug)] -#[cfg(not(target_arch = "x86_64"))] -pub(crate) struct CpuUpExit { - pub(crate) target_cpu: u64, - pub(crate) entry_point: GuestPhysAddr, - pub(crate) arg: u64, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(target_arch = "aarch64")] -pub(crate) struct SendIpiExit { - pub(crate) target_cpu: u64, - pub(crate) target_cpu_aux: u64, - pub(crate) send_to_all: bool, - pub(crate) send_to_self: bool, - pub(crate) vector: u64, -} - -#[allow(dead_code)] -pub(crate) struct VcpuCreateContext { - pub(crate) vcpu_id: usize, - pub(crate) phys_cpu_id: usize, - pub(crate) dtb_addr: Option, - pub(crate) firmware_boot: bool, -} - -#[allow(dead_code)] -pub(crate) struct VcpuSetupContext<'a> { - pub(crate) interrupt_mode: VMInterruptMode, - pub(crate) emulates_console: bool, - pub(crate) passthrough_ports: &'a [PassThroughPortConfig], - pub(crate) memory_regions: &'a [crate::vm::VMMemoryRegion], - pub(crate) firmware_boot: bool, -} - -pub(crate) trait ArchOps { - type VCpu: VmArchVcpuOps; - type PerCpu: VmArchPerCpuOps; - type VcpuCreateState; - type DeferredRunWork; - type NestedPageTable: NestedPageTableOps; - - fn has_hardware_support() -> bool; - - fn max_guest_page_table_levels() -> usize { - 4 - } - - fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { - let mut levels = Self::max_guest_page_table_levels(); - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - levels = levels.min( - crate::percpu::cpu_max_guest_page_table_levels(cpu_id) - .unwrap_or_else(Self::max_guest_page_table_levels), - ); - } - Ok(levels) - } - - fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - let gpa_bits = match levels { - 3 => 39, - 4 => 48, - _ => return ax_errno::ax_err!(InvalidInput, "unsupported nested page-table levels"), - }; - Ok(NestedPagingConfig::new(root_paddr, levels, gpa_bits, 0)) - } - - fn new_nested_page_table(levels: usize) -> AxResult; - - fn clean_dcache_range(_addr: VirtAddr, _size: usize) {} - - fn new_vcpu_create_state( - vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult; - - fn build_vcpu_create_config( - state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig>; - - fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, - ) -> AxResult<::SetupConfig>; - - fn register_platform_irq_injector() {} - - fn vcpu_affinities( - cpu_num: usize, - phys_cpu_ids: Option<&[usize]>, - phys_cpu_sets: Option<&[usize]>, - ) -> Vec<(usize, Option, usize)> { - default_vcpu_affinities(cpu_num, phys_cpu_ids, phys_cpu_sets) - } - - #[cfg(target_arch = "aarch64")] - fn ipi_targets( - vm: &crate::AxVMRef, - current_vcpu_id: usize, - target_cpu: u64, - target_cpu_aux: u64, - send_to_all: bool, - send_to_self: bool, - ) -> CpuMask<64> { - let mut targets = CpuMask::new(); - - if send_to_all { - for vcpu in vm.vcpu_list() { - if vcpu.id() != current_vcpu_id { - targets.set(vcpu.id(), true); - } - } - } else if send_to_self { - targets.set(current_vcpu_id, true); - } else { - let _ = target_cpu_aux; - targets.set(target_cpu as usize, true); - } - - targets - } - - #[expect( - dead_code, - reason = "secondary vCPU boot is unsupported on the x86 build" - )] - fn set_vcpu_on_args(vcpu: &crate::vm::AxVCpuRef, _vcpu_id: usize, arg: usize) { - vcpu.set_gpr(0, arg); - } - - #[expect(dead_code, reason = "CPU-up exits are unsupported on the x86 build")] - fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { - vcpu.set_gpr(0, 0); - } - - #[cfg(target_arch = "x86_64")] - fn set_io_read_result(vcpu: &crate::vm::AxVCpuRef, val: usize) { - vcpu.set_gpr(0, val); - } - - fn before_first_run(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) {} - - fn before_vcpu_run(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) {} - - fn inject_pending_interrupt( - _vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - interrupt: crate::vm::PendingInterrupt, - ) { - match interrupt { - crate::vm::PendingInterrupt::Normal(vector) => { - trace!( - "Injecting queued interrupt {vector:#x} into VM[{}] VCpu[{}]", - vcpu.vm_id(), - vcpu.id() - ); - if let Err(err) = vcpu.inject_interrupt(vector) { - warn!( - "Failed to inject queued interrupt {vector:#x} into VM[{}] VCpu[{}]: \ - {err:?}", - vcpu.vm_id(), - vcpu.id() - ); - } - } - crate::vm::PendingInterrupt::External { - vector, - physical_irq, - } => { - warn!( - "VM[{}] VCpu[{}] dropped unsupported external interrupt vector={vector:#x}, \ - physical_irq={physical_irq:#x}", - vcpu.vm_id(), - vcpu.id() - ); - } - } - } - - fn after_external_interrupt( - _vm: &crate::AxVMRef, - _vcpu: &crate::vm::AxVCpuRef, - vector: usize, - ) { - crate::host::arceos::dispatch_host_irq(vector); - crate::check_timer_events(); - } - - #[cfg(target_arch = "x86_64")] - fn after_preemption_timer(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) { - crate::check_timer_events(); - } - - #[cfg(target_arch = "x86_64")] - fn after_interrupt_end( - _vm: &crate::AxVMRef, - _vcpu: &crate::vm::AxVCpuRef, - _vector: Option, - ) { - } - - #[cfg(target_arch = "loongarch64")] - fn handle_idle(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) { - crate::check_timer_events(); - } - - fn on_last_vcpu_exit(_vm_id: usize) {} - - fn after_mmio_write(_vm: &crate::AxVMRef) {} - - #[cfg(not(target_arch = "x86_64"))] - fn cpu_up_target_vcpu_id(vm: &crate::AxVMRef, target_cpu: u64) -> Option { - vm.get_vcpu_affinities_pcpu_ids() - .iter() - .find_map(|(vcpu_id, _, phys_id)| (*phys_id == target_cpu as usize).then_some(*vcpu_id)) - } - - #[cfg(not(target_arch = "aarch64"))] - fn handle_halt() -> VcpuRunAction { - VcpuRunAction::Wait - } - - fn handle_vcpu_exit_bound( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - exit: ::Exit, - ) -> AxResult>; - - fn finish_deferred_run_work( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - work: Self::DeferredRunWork, - ) -> AxResult; - - fn run_vcpu( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - ) -> AxResult - where - Self: Sized, - { - let vm_id = vm.id(); - let vcpu_id = vcpu.id(); - - match vcpu.state() { - VmVcpuState::Free => vcpu.bind()?, - VmVcpuState::Ready => {} - state => { - return ax_err!( - BadState, - format!("VCpu state is not Free or Ready, but {state:?}") - ); - } - } - - let run_result = vcpu.with_current_cpu_set(|| -> AxResult<_> { - loop { - crate::runtime::vcpus::inject_pending_interrupts::(vm.id(), vcpu_id, vcpu); - - let exit = vcpu.run()?; - trace!("{exit:#x?}"); - match Self::handle_vcpu_exit_bound(vm, vcpu, exit)? { - BoundVcpuExit::Continue => continue, - action => break Ok(action), - } - } - }); - - let unbind_result = vcpu.unbind(); - match run_result { - Ok(BoundVcpuExit::Complete(action)) => { - unbind_result?; - Ok(action) - } - Ok(BoundVcpuExit::Defer(work)) => { - unbind_result?; - Self::finish_deferred_run_work(vm, vcpu, work) - } - Ok(BoundVcpuExit::Continue) => unreachable!("continued exits do not leave run loop"), - Err(err) => { - if let Err(unbind_err) = unbind_result { - warn!( - "VM[{vm_id}] VCpu[{vcpu_id}] unbind after run error failed: {unbind_err:?}" - ); - } - Err(err) - } - } - } -} - -pub(crate) fn handle_mmio_read( - vm: &crate::AxVM, - vcpu: &crate::vm::AxVCpuRef, - exit: MmioReadExit, -) -> AxResult> { - let raw = vm.get_devices()?.handle_mmio_read(exit.addr, exit.width)?; - let masked = raw & crate::vm::width_mask(exit.width); - let val = if exit.signed_ext { - crate::vm::sign_extend_value(masked, exit.width) - } else { - masked & crate::vm::width_mask(exit.reg_width) - }; - vcpu.set_gpr(exit.reg, val); - Ok(BoundVcpuExit::Continue) -} - -pub(crate) fn handle_mmio_write( - vm: &crate::AxVMRef, - exit: MmioWriteExit, -) -> AxResult> { - vm.handle_mmio_write(exit.addr, exit.width, exit.data as usize)?; - A::after_mmio_write(vm); - Ok(BoundVcpuExit::Continue) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn handle_io_read( - vm: &crate::AxVM, - vcpu: &crate::vm::AxVCpuRef, - exit: IoReadExit, -) -> AxResult> { - let val = vm.get_devices()?.handle_port_read(exit.port, exit.width)?; - A::set_io_read_result(vcpu, val); - Ok(BoundVcpuExit::Continue) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn handle_io_write( - vm: &crate::AxVM, - exit: IoWriteExit, -) -> AxResult> { - vm.get_devices()? - .handle_port_write(exit.port, exit.width, exit.data as usize)?; - Ok(BoundVcpuExit::Continue) -} - -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub(crate) fn handle_sys_reg_read( - vm: &crate::AxVM, - vcpu: &crate::vm::AxVCpuRef, - exit: SysRegReadExit, -) -> AxResult> { - let val = vm.get_devices()?.handle_sys_reg_read( - exit.addr, - // System registers are currently modeled as fixed-width device registers. - AccessWidth::Qword, - )?; - vcpu.set_gpr(exit.reg, val); - Ok(BoundVcpuExit::Continue) + CurrentArch::map_address_space(address_space) } -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub(crate) fn handle_sys_reg_write( - vm: &crate::AxVM, - exit: SysRegWriteExit, -) -> AxResult> { - vm.get_devices()? - .handle_sys_reg_write(exit.addr, AccessWidth::Qword, exit.value as usize)?; - Ok(BoundVcpuExit::Continue) -} - -pub(crate) fn handle_hypercall( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - exit: HypercallExit, -) -> AxResult> { - debug!("Hypercall [{:#x}] args {:x?}", exit.nr, exit.args); - match crate::runtime::hvc::HyperCall::new(vm.clone(), exit.nr, exit.args) { - Ok(hypercall) => { - let ret_val = match hypercall.execute() { - Ok(ret_val) => ret_val as isize, - Err(err) => { - warn!("Hypercall [{:#x}] failed: {err:?}", exit.nr); - -1 - } - }; - vcpu.set_return_value(ret_val as usize); - } - Err(err) => { - warn!("Hypercall [{:#x}] failed: {err:?}", exit.nr); - } - } - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) -} - -#[cfg(not(target_arch = "x86_64"))] -pub(crate) fn handle_cpu_up( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - exit: CpuUpExit, -) -> AxResult> { - let vm_id = vm.id(); - let vcpu_id = vcpu.id(); - info!( - "VM[{vm_id}]'s VCpu[{vcpu_id}] try to boot target_cpu [{}] entry_point={:x} arg={:#x}", - exit.target_cpu, exit.entry_point, exit.arg - ); - - let Some(target_vcpu_id) = A::cpu_up_target_vcpu_id(vm, exit.target_cpu) else { - warn!( - "VM[{vm_id}] cannot resolve architecture CPU target {} to a VM-local vCPU", - exit.target_cpu - ); - vcpu.set_return_value(usize::MAX); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); - }; - - match crate::runtime::vcpus::vcpu_on( - vm.clone(), - target_vcpu_id, - exit.entry_point, - exit.arg as _, - ) { - Ok(()) => A::set_cpu_up_success(vcpu), - Err(err) => { - warn!("Failed to boot VM[{vm_id}] VCpu[{target_vcpu_id}]: {err:?}"); - vcpu.set_return_value(usize::MAX); - } - } - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) +pub(crate) fn register_timer_callback() { + CurrentArch::register_timer_callback(); } -#[cfg(target_arch = "aarch64")] -pub(crate) fn handle_send_ipi( - vm: &crate::AxVMRef, - vcpu_id: usize, - exit: SendIpiExit, -) -> AxResult> { - let vm_id = vm.id(); - debug!( - "VM[{vm_id}] run VCpu[{vcpu_id}] SendIPI, target_cpu={:#x}, target_cpu_aux={:#x}, \ - vector={}", - exit.target_cpu, exit.target_cpu_aux, exit.vector - ); - let targets = A::ipi_targets( - vm, - vcpu_id, - exit.target_cpu, - exit.target_cpu_aux, - exit.send_to_all, - exit.send_to_self, - ); - if targets.is_empty() { - warn!( - "VM[{vm_id}] SendIPI has no target: target_cpu={:#x}, target_cpu_aux={:#x}", - exit.target_cpu, exit.target_cpu_aux - ); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); - } - - if targets.get(vcpu_id) { - crate::inject_current_vcpu_interrupt(exit.vector as _) - .expect("failed to inject self IPI into current vCPU"); - } - let mut remote_targets = targets; - remote_targets.set(vcpu_id, false); - if !remote_targets.is_empty() - && let Err(err) = vm.inject_interrupt_to_vcpu(remote_targets, exit.vector as _) - { - warn!( - "Failed to inject interrupt {} to VM[{vm_id}] targets {remote_targets:?}: {err:?}", - exit.vector - ); - } - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) +pub(crate) fn set_oneshot_timer(deadline_ns: u64) { + CurrentArch::set_oneshot_timer(deadline_ns); } -#[cfg(target_arch = "x86_64")] -pub(crate) fn finish_legacy_deferred_run_work( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - work: LegacyDeferredRunWork, -) -> AxResult -where - A: ArchOps, -{ - match work { - LegacyDeferredRunWork::ExternalInterrupt { vector } => { - A::after_external_interrupt(vm, vcpu, vector); - } - LegacyDeferredRunWork::PreemptionTimer => { - A::after_preemption_timer(vm, vcpu); - } - LegacyDeferredRunWork::InterruptEnd { vector } => { - A::after_interrupt_end(vm, vcpu, vector); - } - } - Ok(VcpuRunAction::Yield) +pub(crate) fn load_images_from_memory( + loader: &mut crate::boot::images::ImageLoaderCore<'_>, + images: crate::boot::StaticVmImage, +) -> AxResult { + CurrentArch::load_images_from_memory(loader, images) } -pub(crate) fn target_phys_cpu_ids(vcpu_mappings: &[(usize, Option, usize)]) -> Vec { - let mut cpu_ids = Vec::new(); - for (_, maybe_mask, phys_id) in vcpu_mappings { - if let Some(mask) = maybe_mask { - for cpu_id in 0..usize::BITS as usize { - if mask & (1usize << cpu_id) != 0 && !cpu_ids.contains(&cpu_id) { - cpu_ids.push(cpu_id); - } - } - } else if !cpu_ids.contains(phys_id) { - cpu_ids.push(*phys_id); - } - } - cpu_ids +#[cfg(any(feature = "fs", feature = "host-fs"))] +pub(crate) fn load_images_from_filesystem( + loader: &mut crate::boot::images::ImageLoaderCore<'_>, +) -> AxResult { + CurrentArch::load_images_from_filesystem(loader) } -pub(crate) fn default_vcpu_affinities( - cpu_num: usize, - phys_cpu_ids: Option<&[usize]>, - phys_cpu_sets: Option<&[usize]>, -) -> Vec<(usize, Option, usize)> { - let mut vcpus = Vec::with_capacity(cpu_num); - for vcpu_id in 0..cpu_num { - vcpus.push((vcpu_id, None, vcpu_id)); - } - - if let Some(phys_cpu_sets) = phys_cpu_sets { - for (vcpu_id, pcpu_mask_bitmap) in phys_cpu_sets.iter().enumerate() { - if let Some(vcpu) = vcpus.get_mut(vcpu_id) { - vcpu.1 = Some(*pcpu_mask_bitmap); - } - } - } - - if let Some(phys_cpu_ids) = phys_cpu_ids { - for (vcpu_id, phys_id) in phys_cpu_ids.iter().enumerate() { - if let Some(vcpu) = vcpus.get_mut(vcpu_id) { - vcpu.2 = *phys_id; - } - } - } - - vcpus +pub(crate) fn is_x86_linux_image_config( + config: &axvmconfig::AxVMCrateConfig, + provider: &dyn crate::boot::BootImageProvider, +) -> bool { + CurrentArch::is_x86_linux_image_config(config, provider) } diff --git a/virtualization/axvm/src/arch/riscv64/capabilities.rs b/virtualization/axvm/src/arch/riscv64/capabilities.rs new file mode 100644 index 0000000000..0fc0d173df --- /dev/null +++ b/virtualization/axvm/src/arch/riscv64/capabilities.rs @@ -0,0 +1,42 @@ +//! RISC-V implementations of AxVM platform capability hooks. + +use ax_errno::{AxResult, ax_err_type}; + +use super::Riscv64Arch; +use crate::architecture::{ + AddressSpacePlatform, BootImagePlatform, DevicePlatform, HostTimePlatform, +}; + +impl DevicePlatform for Riscv64Arch { + fn configure_interrupt_fabric( + factories: &mut axdevice::DeviceFactoryRegistry, + mode: axvm_types::VMInterruptMode, + configs: &[axvm_types::EmulatedDeviceConfig], + ) -> AxResult { + super::irq::configure(factories, mode, configs) + } +} + +impl AddressSpacePlatform for Riscv64Arch {} + +impl HostTimePlatform for Riscv64Arch {} + +impl BootImagePlatform for Riscv64Arch { + fn load_guest_dtb( + loader: &crate::boot::images::ImageLoaderCore<'_>, + dtb: &crate::boot::fdt::GuestDtbImage, + ) -> AxResult { + let bytes = dtb.as_bytes(); + let source = core::ptr::NonNull::new(bytes.as_ptr() as *mut u8) + .ok_or_else(|| ax_err_type!(InvalidData, "Guest DTB pointer is null"))?; + crate::boot::fdt::update_fdt(source, bytes.len(), loader.vm.clone(), &loader.config) + } +} + +pub fn host_fdt_bootarg() -> usize { + ax_std::os::arceos::modules::ax_hal::dtb::get_bootarg() +} + +pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + ax_std::os::arceos::modules::ax_hal::mem::phys_to_virt(paddr) +} diff --git a/virtualization/axvm/src/arch/riscv64/images.rs b/virtualization/axvm/src/arch/riscv64/images.rs new file mode 100644 index 0000000000..5a22968ad6 --- /dev/null +++ b/virtualization/axvm/src/arch/riscv64/images.rs @@ -0,0 +1,33 @@ +//! Public RISC-V image-loader facade preserving the DTB constructor contract. + +use ax_errno::AxResult; +use axvmconfig::AxVMCrateConfig; + +use crate::{ + AxVMRef, VMMemoryRegion, + boot::{BootImageProvider, fdt::GuestDtbImage, images::ImageLoaderCore}, +}; + +pub struct ImageLoader<'a>(ImageLoaderCore<'a>); + +impl<'a> ImageLoader<'a> { + pub fn new( + main_memory: VMMemoryRegion, + config: AxVMCrateConfig, + vm: AxVMRef, + provider: &'a dyn BootImageProvider, + guest_dtb: Option, + ) -> Self { + Self(ImageLoaderCore::new( + main_memory, + config, + vm, + provider, + guest_dtb, + )) + } + + pub fn load(&mut self) -> AxResult { + self.0.load() + } +} diff --git a/virtualization/axvm/src/arch/riscv64/mod.rs b/virtualization/axvm/src/arch/riscv64/mod.rs index 72226a0658..aa0e73bde0 100644 --- a/virtualization/axvm/src/arch/riscv64/mod.rs +++ b/virtualization/axvm/src/arch/riscv64/mod.rs @@ -15,18 +15,26 @@ use riscv_vcpu::{ use riscv_vplic::host::RiscvVplicHostIf; use super::{ - ArchOps, BoundVcpuExit, CpuUpExit, HypercallExit, MmioReadExit, MmioWriteExit, - VcpuCreateContext, VcpuRunAction, VcpuSetupContext, default_vcpu_affinities, - target_phys_cpu_ids, + ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, + VcpuRunAction, VcpuSetupContext, }; use crate::{ StopReason, + architecture::ops::{default_vcpu_affinities, target_phys_cpu_ids}, host::{HostMemory, default_host}, }; +mod capabilities; +#[path = "../../architecture/cpu_up.rs"] +mod cpu_up; +mod images; mod irq; mod npt; +pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; +use cpu_up::{CpuUpExit, CpuUpOps}; +pub use images::ImageLoader; + pub(crate) struct Riscv64Arch; #[derive(Clone, Copy, Debug)] @@ -34,6 +42,12 @@ pub(crate) enum RiscvDeferredRunWork { ExternalInterrupt { vector: usize }, } +impl CpuUpOps for Riscv64Arch { + fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { + vcpu.set_gpr(RiscvGprIndex::A0 as usize, 0); + } +} + impl ArchOps for Riscv64Arch { type VCpu = AxvmRiscvVcpu; type PerCpu = AxvmRiscvPerCpu; @@ -85,15 +99,17 @@ impl ArchOps for Riscv64Arch { _state: &Self::VcpuCreateState, ctx: VcpuCreateContext, ) -> AxResult<::CreateConfig> { + let (vcpu_id, _phys_cpu_id, dtb_addr, _firmware_boot) = ctx.into_parts(); Ok(RiscvVcpuCreateConfig { - hart_id: ctx.vcpu_id, - dtb_addr: ctx.dtb_addr.unwrap_or_default().as_usize(), + hart_id: vcpu_id, + dtb_addr: dtb_addr.unwrap_or_default().as_usize(), }) } fn build_vcpu_setup_config( - _ctx: VcpuSetupContext<'_>, + ctx: VcpuSetupContext<'_>, ) -> AxResult<::SetupConfig> { + let _ = ctx.into_parts(); Ok(()) } @@ -114,7 +130,7 @@ impl ArchOps for Riscv64Arch { return; }; let irq_sources = vm.with_config(|config| config.pass_through_irqs().to_vec()); - irq::set_virtual_irq_targets(cpu_id, &irq_sources); + crate::irq::set_riscv_virtual_irq_targets(cpu_id, &irq_sources); } fn vcpu_affinities( @@ -136,10 +152,6 @@ impl ArchOps for Riscv64Arch { vcpu.set_gpr(RiscvGprIndex::A1 as usize, arg); } - fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { - vcpu.set_gpr(RiscvGprIndex::A0 as usize, 0); - } - fn after_external_interrupt( _vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef, @@ -201,7 +213,7 @@ impl ArchOps for Riscv64Arch { target_cpu, entry_point, arg, - } => super::handle_cpu_up::( + } => cpu_up::handle::( vm, vcpu, CpuUpExit { @@ -216,19 +228,29 @@ impl ArchOps for Riscv64Arch { vm.id(), vcpu.id() ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Wait)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: true, + stop_reason: None, + })) } RiscvVmExit::Halt => { debug!("VM[{}] run VCpu[{}] Halt", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(Self::handle_halt())) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: true, + stop_reason: None, + })) } RiscvVmExit::SystemDown => { warn!("VM[{}] run VCpu[{}] SystemDown", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Stop( - StopReason::SystemDown, - ))) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: Some(StopReason::SystemDown), + })) } - RiscvVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)), + RiscvVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })), } } @@ -242,7 +264,10 @@ impl ArchOps for Riscv64Arch { Self::after_external_interrupt(vm, vcpu, vector); } } - Ok(VcpuRunAction::Yield) + Ok(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }) } } @@ -261,7 +286,10 @@ fn handle_riscv_nested_page_fault( vcpu.id(), ax_addr.as_usize() ); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); + return Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })); }; return Riscv64Arch::handle_vcpu_exit_bound(vm, vcpu, decoded); } @@ -277,7 +305,10 @@ fn handle_riscv_nested_page_fault( ax_addr.as_usize(), ax_flags ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } } @@ -460,7 +491,7 @@ impl RiscvVplicHostIf for RiscvVplicHostIfImpl { } fn register_platform_irq_injector() { - irq::register_virtual_irq_injector(inject_virtual_irq); + crate::irq::register_riscv_virtual_irq_injector(inject_virtual_irq); } fn first_cpu_in_mask(mask: usize) -> Option { diff --git a/virtualization/axvm/src/arch/riscv64/npt.rs b/virtualization/axvm/src/arch/riscv64/npt.rs index 631f9aab58..51b394d6f4 100644 --- a/virtualization/axvm/src/arch/riscv64/npt.rs +++ b/virtualization/axvm/src/arch/riscv64/npt.rs @@ -106,4 +106,4 @@ impl ptg::PageTableEntry for RiscvPte { } pub(crate) type NestedPageTable = - crate::arch::npt::LeveledPageTable; + crate::npt::LeveledPageTable; diff --git a/virtualization/axvm/src/arch/x86_64/boot/images/x86/boot_params.rs b/virtualization/axvm/src/arch/x86_64/boot/boot_params.rs similarity index 99% rename from virtualization/axvm/src/arch/x86_64/boot/images/x86/boot_params.rs rename to virtualization/axvm/src/arch/x86_64/boot/boot_params.rs index 1906cdba4a..132c7d2b4c 100644 --- a/virtualization/axvm/src/arch/x86_64/boot/images/x86/boot_params.rs +++ b/virtualization/axvm/src/arch/x86_64/boot/boot_params.rs @@ -354,8 +354,10 @@ fn write_u64(buffer: &mut [u8], offset: usize, value: u64) { #[cfg(test)] mod tests { - use super::*; - use crate::boot::images::x86::linux::{BOOT_PARAMS_GPA, BOOT_STUB_GPA, BOOT_STUB_SIZE}; + use super::{ + super::linux::{BOOT_PARAMS_GPA, BOOT_STUB_GPA, BOOT_STUB_SIZE}, + *, + }; const SETUP_SECTS_OFFSET: usize = 0x1f1; const BOOT_FLAG_OFFSET: usize = 0x1fe; diff --git a/virtualization/axvm/src/arch/x86_64/boot/images/mod.rs b/virtualization/axvm/src/arch/x86_64/boot/images/mod.rs deleted file mode 100644 index 5bcc1cac7b..0000000000 --- a/virtualization/axvm/src/arch/x86_64/boot/images/mod.rs +++ /dev/null @@ -1,1185 +0,0 @@ -// Copyright 2025 The Axvisor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use alloc::format; - -use ax_errno::{AxResult, ax_err, ax_err_type}; -use axvmconfig::AxVMCrateConfig; -#[cfg(target_arch = "x86_64")] -use axvmconfig::EmulatedDeviceType; -#[cfg(any(target_arch = "loongarch64", target_arch = "x86_64"))] -use axvmconfig::VMBootProtocol; -#[cfg(target_arch = "x86_64")] -use axvmconfig::VmMemMappingType; -use byte_unit::Byte; - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use crate::boot::fdt::GuestDtbImage; -use crate::{ - AxVMRef, GuestPhysAddr, VMMemoryRegion, - boot::{BootImageProvider, StaticVmImage}, -}; - -mod linux; -#[cfg(target_arch = "x86_64")] -mod x86; -#[cfg(target_arch = "x86_64")] -use x86::boot_params as x86_boot_params; -#[cfg(target_arch = "x86_64")] -use x86::linux as x86_linux; -#[cfg(target_arch = "x86_64")] -use x86::linux_boot as x86_linux_boot; -#[cfg(target_arch = "x86_64")] -use x86::mptable as x86_mptable; -#[cfg(target_arch = "x86_64")] -use x86::multiboot as x86_boot; - -#[cfg(target_arch = "x86_64")] -pub fn is_x86_linux_image_config( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> bool { - if !should_direct_boot_x86_linux(config) { - return false; - } - - match config.kernel.image_location.as_deref() { - Some("memory") => with_memory_image(config, provider, detect_x86_linux_image).is_some(), - #[cfg(any(feature = "fs", feature = "host-fs"))] - Some("fs") => fs::kernel_read(config, provider, x86_linux::HEADER_READ_SIZE) - .ok() - .and_then(|data| detect_x86_linux_image(&data)) - .is_some(), - _ => false, - } -} - -#[cfg(target_arch = "x86_64")] -pub const fn x86_qemu_passthrough_block_intx() -> (u8, u8, u8, usize) { - ( - x86_mptable::QEMU_PASSTHROUGH_BLOCK_DEVICE, - x86_mptable::QEMU_PASSTHROUGH_BLOCK_FUNCTION, - x86_mptable::QEMU_PASSTHROUGH_BLOCK_PIN, - x86_mptable::qemu_passthrough_block_gsi(), - ) -} - -pub fn get_image_header( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> Option { - match config.kernel.image_location.as_deref() { - Some("memory") => with_memory_image(config, provider, linux::Header::parse).flatten(), - #[cfg(any(feature = "fs", feature = "host-fs"))] - Some("fs") => { - let read_size = linux::Header::hdr_size(); - let data = fs::kernel_read(config, provider, read_size).ok()?; - linux::Header::parse(&data) - } - _ => None, - } -} - -fn with_memory_image( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, - func: F, -) -> Option -where - F: FnOnce(&[u8]) -> R, -{ - let vm_images = provider - .static_vm_images() - .iter() - .find(|&v| v.id == config.base.id)?; - - Some(func(vm_images.kernel)) -} - -fn memory_images_for_vm( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> AxResult { - provider - .static_vm_images() - .iter() - .copied() - .find(|&v| v.id == config.base.id) - .ok_or_else(|| { - ax_err_type!( - NotFound, - "VM images are missing; pass VM configs with AXVISOR_VM_CONFIGS" - ) - }) -} - -#[cfg(target_arch = "loongarch64")] -fn provider_firmware_image_for_vm( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> Option<&'static [u8]> { - provider - .static_firmware_images() - .iter() - .find(|&v| v.id == config.base.id) - .map(|v| v.bios) - .flatten() -} - -pub struct ImageLoader<'a> { - provider: &'a dyn BootImageProvider, - main_memory: VMMemoryRegion, - vm: AxVMRef, - config: AxVMCrateConfig, - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - guest_dtb: Option, - kernel_load_gpa: GuestPhysAddr, - bios_load_gpa: Option, - ramdisk_load_gpa: Option, -} - -impl<'a> ImageLoader<'a> { - pub fn new( - main_memory: VMMemoryRegion, - config: AxVMCrateConfig, - vm: AxVMRef, - provider: &'a dyn BootImageProvider, - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] guest_dtb: Option< - GuestDtbImage, - >, - ) -> Self { - Self { - provider, - main_memory, - vm, - config, - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - guest_dtb, - kernel_load_gpa: GuestPhysAddr::default(), - bios_load_gpa: None, - ramdisk_load_gpa: None, - } - } - - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - fn load_guest_dtb(&self, dtb: &GuestDtbImage) -> AxResult { - let bytes = dtb.as_bytes(); - let dtb_src = core::ptr::NonNull::new(bytes.as_ptr() as *mut u8) - .ok_or_else(|| ax_err_type!(InvalidData, "Guest DTB pointer is null"))?; - crate::boot::fdt::update_fdt(dtb_src, bytes.len(), self.vm.clone(), &self.config) - } - - pub fn load(&mut self) -> AxResult { - self.config.kernel.validate_boot_config()?; - debug!( - "Loading VM[{}] images into memory region: gpa={:#x}, hva={:#x}, size={:#}", - self.vm.id(), - self.main_memory.gpa, - self.main_memory.hva, - Byte::from(self.main_memory.size()) - ); - - self.vm.with_config(|config| { - self.kernel_load_gpa = config.image_config.kernel_load_gpa; - self.bios_load_gpa = config.image_config.bios_load_gpa; - self.ramdisk_load_gpa = config.image_config.ramdisk.as_ref().map(|r| r.load_gpa); - }); - - match self.config.kernel.image_location.as_deref() { - Some("memory") => self.load_vm_images_from_memory(), - #[cfg(any(feature = "fs", feature = "host-fs"))] - Some("fs") => fs::load_vm_images_from_filesystem(self), - _ => ax_err!( - InvalidInput, - "Unsupported image_location; use \"memory\" or enable fs feature for \"fs\"" - ), - } - } - - /// Load VM images from memory - /// into the guest VM's memory space based on the VM configuration. - fn load_vm_images_from_memory(&mut self) -> AxResult { - debug!("Loading VM[{}] images from memory", self.config.base.id); - - let vm_images = memory_images_for_vm(&self.config, self.provider)?; - - #[cfg(target_arch = "x86_64")] - if should_direct_boot_x86_linux(&self.config) - && let Some(header) = detect_x86_linux_image(vm_images.kernel) - { - return self.load_x86_linux_images_from_memory( - header, - vm_images.kernel, - vm_images.ramdisk, - ); - } - - #[cfg(target_arch = "loongarch64")] - if self.loongarch_uefi_boot() { - self.load_loongarch_uefi_firmware_dtb()?; - self.add_loongarch_uefi_fw_cfg_from_memory(vm_images.kernel, vm_images.ramdisk)?; - self.load_loongarch_uefi_firmware_from_memory(vm_images.bios)?; - return Ok(()); - } - - load_vm_image_from_memory(vm_images.kernel, self.kernel_load_gpa, self.vm.clone())?; - - // Load Ramdisk image and record its size before regenerating the DTB. - if let Some(buffer) = vm_images.ramdisk { - self.load_ramdisk_from_memory(buffer)?; - } - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - { - // Load DTB image - if let Some(dtb) = &self.guest_dtb { - self.load_guest_dtb(dtb)?; - } else { - info!("VM[{}] has no guest DTB image to load", self.config.base.id); - } - } - - self.load_boot_image_from_memory(vm_images.bios)?; - - Ok(()) - } - - #[cfg(target_arch = "x86_64")] - fn load_x86_linux_images_from_memory( - &mut self, - header: x86_linux::X86LinuxHeader, - kernel: &[u8], - ramdisk: Option<&[u8]>, - ) -> AxResult { - self.adjust_x86_linux_dma_identity_layout()?; - let payload = x86_linux_payload(&header, kernel)?; - let initrd = if let Some(ramdisk) = ramdisk { - Some(x86_linux::X86LinuxRange::new( - self.ramdisk_load_gpa()?.as_usize(), - ramdisk.len(), - )) - } else { - None - }; - let layout = x86_linux::X86LinuxLoadLayout::new( - &header, - self.kernel_load_gpa.as_usize(), - payload.len(), - initrd, - ) - .map_err(x86_linux_layout_error)?; - - self.load_x86_linux_layout(header, layout, kernel)?; - load_vm_image_from_memory(payload, self.kernel_load_gpa, self.vm.clone())?; - - if let Some(buffer) = ramdisk { - self.load_ramdisk_from_memory(buffer)?; - } - - Ok(()) - } - - fn load_boot_image_from_memory(&self, bios: Option<&[u8]>) -> AxResult { - if !self.config.kernel.enable_bios { - return Ok(()); - } - - if let Some(buffer) = bios { - let load_gpa = self - .bios_load_gpa - .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; - load_vm_image_from_memory(buffer, load_gpa, self.vm.clone())?; - #[cfg(target_arch = "x86_64")] - if should_patch_x86_multiboot_info(&self.config) { - self.load_x86_multiboot_info(buffer, load_gpa)?; - } - return Ok(()); - } - - #[cfg(target_arch = "x86_64")] - if self.config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { - let firmware_path = self.config.kernel.boot_firmware_path().ok_or_else(|| { - ax_errno::ax_err_type!(NotFound, "UEFI firmware image path is missed") - })?; - let load_gpa = self.bios_load_gpa.ok_or_else(|| { - ax_errno::ax_err_type!(NotFound, "UEFI firmware load addr is missed") - })?; - #[cfg(not(any(feature = "fs", feature = "host-fs")))] - let _ = (firmware_path, load_gpa); - - #[cfg(any(feature = "fs", feature = "host-fs"))] - { - info!( - "Loading UEFI firmware image {} at GPA {:#x}", - firmware_path, - load_gpa.as_usize() - ); - return fs::load_vm_image(firmware_path, load_gpa, self.vm.clone(), self.provider); - } - - #[cfg(not(any(feature = "fs", feature = "host-fs")))] - { - return Err(ax_errno::ax_err_type!( - Unsupported, - "UEFI firmware path requires the fs feature when no firmware image buffer is \ - available" - )); - } - } - - #[cfg(target_arch = "x86_64")] - if self.should_load_default_x86_boot_image() { - let bios_load_gpa = builtin_x86_bios_load_gpa(self.bios_load_gpa)?; - info!( - "Loading built-in x86 boot image at GPA {:#x}", - bios_load_gpa.as_usize() - ); - load_vm_image_from_memory( - x86_boot::DEFAULT_BIOS_IMAGE, - bios_load_gpa, - self.vm.clone(), - )?; - #[cfg(target_arch = "x86_64")] - self.load_x86_multiboot_info(x86_boot::DEFAULT_BIOS_IMAGE, bios_load_gpa)?; - } - - Ok(()) - } - - #[cfg(target_arch = "loongarch64")] - fn loongarch_uefi_boot(&self) -> bool { - self.config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi - } - - #[cfg(target_arch = "loongarch64")] - fn add_loongarch_uefi_fw_cfg( - &self, - kernel: &'static [u8], - ramdisk: Option<&'static [u8]>, - ) -> AxResult { - let fw_cfg = crate::boot::guest_platform::loongarch64::emulated_fw_cfg(&self.config)?; - - self.vm.add_fw_cfg_device(crate::FwCfgDeviceConfig { - base: GuestPhysAddr::from(fw_cfg.base_gpa), - size: fw_cfg.length, - kernel, - initrd: ramdisk, - cmdline: self.config.kernel.cmdline.clone(), - cpu_num: self.config.base.cpu_num as u16, - platform: crate::boot::guest_platform::loongarch64::fw_cfg_platform_config( - &self.vm, - &self.config, - ), - }) - } - - #[cfg(target_arch = "loongarch64")] - fn add_loongarch_uefi_fw_cfg_from_memory( - &self, - kernel: &'static [u8], - ramdisk: Option<&'static [u8]>, - ) -> AxResult { - debug!( - "VM[{}] configuring LoongArch UEFI fw_cfg payloads from memory: kernel={} bytes, \ - ramdisk={:?}", - self.config.base.id, - kernel.len(), - ramdisk.map(|data| data.len()) - ); - self.add_loongarch_uefi_fw_cfg(kernel, ramdisk) - } - - #[cfg(target_arch = "loongarch64")] - fn load_loongarch_uefi_firmware_from_memory(&self, bios: Option<&[u8]>) -> AxResult { - let firmware = bios - .or_else(|| provider_firmware_image_for_vm(&self.config, self.provider)) - .ok_or_else(|| { - ax_err_type!( - NotFound, - "LoongArch UEFI boot requires a build-time firmware image" - ) - })?; - self.load_loongarch_uefi_firmware_image(firmware) - } - - #[cfg(target_arch = "loongarch64")] - fn load_loongarch_uefi_firmware_image(&self, firmware: &[u8]) -> AxResult { - let load_gpa = self - .bios_load_gpa - .ok_or_else(|| ax_err_type!(NotFound, "LoongArch UEFI firmware load addr is missed"))?; - let flash_len = self - .config - .kernel - .memory_regions - .iter() - .find(|region| region.gpa == load_gpa.as_usize()) - .map_or(firmware.len(), |region| region.size); - fill_vm_region(load_gpa, flash_len, 0xff, self.vm.clone())?; - load_vm_image_from_memory(firmware, load_gpa, self.vm.clone()) - } - - #[cfg(target_arch = "loongarch64")] - fn load_loongarch_uefi_firmware_dtb(&self) -> AxResult { - crate::boot::guest_platform::loongarch64::prepare_uefi_runtime_config( - &self.vm, - &self.config, - ); - crate::boot::guest_platform::loongarch64::load_firmware_fdt(&self.vm, &self.config) - } - - #[cfg(target_arch = "x86_64")] - fn should_load_default_x86_boot_image(&self) -> bool { - self.config.kernel.enable_bios - && self.config.kernel.boot_firmware_path().is_none() - && self.config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot - } - - #[cfg(target_arch = "x86_64")] - fn load_x86_multiboot_info(&self, bios_image: &[u8], bios_load_gpa: GuestPhysAddr) -> AxResult { - const MULTIBOOT_INFO_GPA: usize = 0x6000; - const MULTIBOOT_MMAP_GPA: usize = 0x6040; - const MULTIBOOT_INFO_FLAGS: u32 = (1 << 0) | (1 << 6); - const MULTIBOOT_MEMORY_AVAILABLE: u32 = 1; - - let mem_base = self.main_memory.gpa.as_usize() as u64; - let mem_size = self.main_memory.size() as u64; - let mem_upper_kb = mem_size.saturating_sub(0x100000) / 1024; - - let mut mbi = [0u8; 52]; - write_u32(&mut mbi, 0, MULTIBOOT_INFO_FLAGS); - write_u32(&mut mbi, 4, 639); - write_u32(&mut mbi, 8, mem_upper_kb as u32); - write_u32(&mut mbi, 44, 24); - write_u32(&mut mbi, 48, MULTIBOOT_MMAP_GPA as u32); - - let mut mmap = [0u8; 24]; - write_u32(&mut mmap, 0, 20); - write_u64(&mut mmap, 4, mem_base); - write_u64(&mut mmap, 12, mem_size); - write_u32(&mut mmap, 20, MULTIBOOT_MEMORY_AVAILABLE); - - let mbi_gpa = (MULTIBOOT_INFO_GPA as u32).to_le_bytes(); - validate_x86_bios_patch_region(bios_image)?; - load_vm_image_from_memory(&mbi, MULTIBOOT_INFO_GPA.into(), self.vm.clone())?; - load_vm_image_from_memory(&mmap, MULTIBOOT_MMAP_GPA.into(), self.vm.clone())?; - load_vm_image_from_memory( - &mbi_gpa, - (bios_load_gpa.as_usize() + x86_boot::AXVM_BIOS_EBX_IMM_OFFSET).into(), - self.vm.clone(), - )?; - Ok(()) - } - - fn load_ramdisk_from_memory(&self, ramdisk: &[u8]) -> AxResult { - let load_gpa = self.ramdisk_load_gpa()?; - let size = ramdisk.len(); - self.vm.with_config(|config| { - if let Some(ref mut rd) = config.image_config.ramdisk { - rd.size = Some(size); - } - }); - info!( - "Loading ramdisk image from memory ({} bytes) into GPA @{:#x}", - size, - load_gpa.as_usize() - ); - load_vm_image_from_memory(ramdisk, load_gpa, self.vm.clone()) - } - - fn ramdisk_load_gpa(&self) -> AxResult { - self.ramdisk_load_gpa - .ok_or_else(|| ax_errno::ax_err_type!(NotFound, "Ramdisk load addr is missed")) - } - - #[cfg(target_arch = "x86_64")] - fn adjust_x86_linux_dma_identity_layout(&mut self) -> AxResult { - if !self.main_memory.is_identical() { - return Ok(()); - } - - let memory_base = self.main_memory.gpa.as_usize(); - let configured_kernel = self.config.kernel.kernel_load_addr; - let configured_ramdisk = self.config.kernel.ramdisk_load_addr; - - self.kernel_load_gpa = GuestPhysAddr::from(memory_base + configured_kernel); - if let Some(ramdisk_load_addr) = configured_ramdisk { - self.ramdisk_load_gpa = Some(GuestPhysAddr::from(memory_base + ramdisk_load_addr)); - } - - self.vm.with_config(|config| { - config.image_config.kernel_load_gpa = self.kernel_load_gpa; - if let Some(load_gpa) = self.ramdisk_load_gpa - && let Some(ref mut ramdisk) = config.image_config.ramdisk - { - ramdisk.load_gpa = load_gpa; - } - }); - - info!( - "Adjusted x86 Linux identity DMA layout for VM[{}]: memory_base={:#x}, \ - kernel_load_gpa={:#x}, ramdisk_load_gpa={:?}", - self.vm.id(), - memory_base, - self.kernel_load_gpa.as_usize(), - self.ramdisk_load_gpa - ); - Ok(()) - } - - #[cfg(target_arch = "x86_64")] - fn load_x86_linux_layout( - &self, - header: x86_linux::X86LinuxHeader, - layout: x86_linux::X86LinuxLoadLayout, - kernel: &[u8], - ) -> AxResult { - info!( - "x86 Linux layout for VM[{}]: header={:#x?}, payload_offset={:#x}, \ - boot_params=[{:#x}..{:#x}), boot_stub=[{:#x}..{:#x}), kernel=[{:#x}..{:#x}), \ - initrd={:?}", - self.config.base.id, - header, - header.payload_offset(), - layout.boot_params.start, - layout.boot_params.end().unwrap(), - layout.boot_stub.start, - layout.boot_stub.end().unwrap(), - layout.kernel.start, - layout.kernel.end().unwrap(), - layout.initrd - ); - - let boot_params = self.build_x86_boot_params(header, layout, kernel)?; - let boot_stub = self.build_x86_linux_boot_stub(&layout)?; - let mp_table = x86_mptable::build(); - load_vm_image_from_memory( - &boot_params, - layout.boot_params.start.into(), - self.vm.clone(), - )?; - load_vm_image_from_memory(&boot_stub, layout.boot_stub.start.into(), self.vm.clone())?; - load_vm_image_from_memory(&mp_table, x86_mptable::MP_TABLE_GPA.into(), self.vm.clone())?; - self.install_x86_linux_boot_entry(&layout); - Ok(()) - } - - #[cfg(target_arch = "x86_64")] - fn build_x86_linux_boot_stub( - &self, - layout: &x86_linux::X86LinuxLoadLayout, - ) -> AxResult<[u8; x86_linux::BOOT_STUB_SIZE]> { - x86_linux_boot::build_boot_image(layout).map_err(|err| { - ax_errno::ax_err_type!( - InvalidInput, - format!("failed to build x86 Linux boot stub: {err:?}") - ) - }) - } - - #[cfg(target_arch = "x86_64")] - fn install_x86_linux_boot_entry(&self, layout: &x86_linux::X86LinuxLoadLayout) { - let entry = GuestPhysAddr::from(x86_linux_boot::DEFAULT_LINUX_BOOT_LOAD_GPA); - self.vm.with_config(|config| { - config.cpu_config.bsp_entry = entry; - config.cpu_config.ap_entry = entry; - }); - info!( - "x86 Linux direct boot entry for VM[{}]: stub={:#x}, boot_params={:#x}, \ - kernel_entry={:#x}, initrd={:?}", - self.config.base.id, - layout.boot_stub.start, - layout.boot_params.start, - layout.kernel.start, - layout.initrd - ); - } - - #[cfg(target_arch = "x86_64")] - fn build_x86_boot_params( - &self, - header: x86_linux::X86LinuxHeader, - layout: x86_linux::X86LinuxLoadLayout, - kernel: &[u8], - ) -> AxResult<[u8; x86_linux::BOOT_PARAMS_SIZE]> { - let mut builder = x86_boot_params::BootParamsBuilder::new( - kernel, - header, - layout, - x86_linux::X86LinuxRange::new(self.main_memory.gpa.as_usize(), self.main_memory.size()), - ); - let command_line = self.config.kernel.cmdline.as_deref().ok_or_else(|| { - ax_errno::ax_err_type!( - InvalidInput, - "x86 Linux direct boot requires kernel.cmdline in the VM config" - ) - })?; - builder.set_command_line(command_line).map_err(|err| { - ax_errno::ax_err_type!( - InvalidInput, - format!("invalid x86 Linux command line: {err:?}") - ) - })?; - - for memory in &self.config.kernel.memory_regions { - if memory.map_type == VmMemMappingType::MapAlloc { - builder.add_ram_range(x86_linux::X86LinuxRange::new(memory.gpa, memory.size)); - } - } - - for device in &self.config.devices.passthrough_devices { - builder.add_reserved_range(x86_linux::X86LinuxRange::new( - device.base_gpa, - device.length, - )); - } - for address in &self.config.devices.passthrough_addresses { - builder.add_reserved_range(x86_linux::X86LinuxRange::new( - address.base_gpa, - address.length, - )); - } - for device in &self.config.devices.emu_devices { - if matches!(device.emu_type, EmulatedDeviceType::X86IoApic) { - builder.add_reserved_range(x86_linux::X86LinuxRange::new( - device.base_gpa, - device.length, - )); - } - } - builder.add_reserved_range(x86_mptable::reserved_range()); - - builder.build().map_err(|err| { - ax_errno::ax_err_type!( - InvalidInput, - format!("failed to build x86 boot_params: {err:?}") - ) - }) - } - - #[cfg(any(feature = "fs", feature = "host-fs"))] - fn load_ramdisk_from_filesystem(&self, ramdisk_path: &str) -> AxResult { - let load_gpa = self - .vm - .with_config(|config| config.image_config.ramdisk.as_ref().map(|r| r.load_gpa)) - .ok_or_else(|| ax_errno::ax_err_type!(NotFound, "Ramdisk load addr is missed"))?; - let ramdisk_size = fs::image_size(ramdisk_path, self.provider)?; - self.vm.with_config(|config| { - if let Some(ref mut rd) = config.image_config.ramdisk { - rd.size = Some(ramdisk_size); - } - }); - info!( - "Loading ramdisk image from filesystem {} ({} bytes) into GPA @{:#x}", - ramdisk_path, - ramdisk_size, - load_gpa.as_usize() - ); - fs::load_vm_image(ramdisk_path, load_gpa, self.vm.clone(), self.provider) - } -} - -pub fn load_vm_image_from_memory( - image_buffer: &[u8], - load_addr: GuestPhysAddr, - vm: AxVMRef, -) -> AxResult { - let mut buffer_pos = 0; - - let image_size = image_buffer.len(); - - debug!( - "loading VM image from memory {:?} {}", - load_addr, - image_buffer.len() - ); - - let image_load_regions = vm.get_image_load_region(load_addr, image_size)?; - - for region in image_load_regions { - let region_len = region.len(); - let bytes_to_write = region_len.min(image_size - buffer_pos); - - // SAFETY: `region` is valid writable guest memory obtained from - // `vm.get_image_load_region()`; `bytes_to_write <= region.len()` is - // guaranteed by `region_len.min(image_size - buffer_pos)`; and - // `image_buffer[buffer_pos..]` has at least `bytes_to_write` bytes. - // The source and destination do not overlap (guest HPA vs host image buffer). - unsafe { - core::ptr::copy_nonoverlapping( - image_buffer[buffer_pos..].as_ptr(), - region.as_mut_ptr().cast(), - bytes_to_write, - ); - } - - crate::clean_dcache_range((region.as_ptr() as usize).into(), bytes_to_write); - - // Update the position of the buffer. - buffer_pos += bytes_to_write; - - // If the buffer is fully written, exit the loop. - if buffer_pos >= image_size { - debug!("copy size: {bytes_to_write}"); - break; - } - } - - if buffer_pos == image_size { - Ok(()) - } else { - ax_err!( - InvalidData, - format!("VM image was only partially loaded: {buffer_pos}/{image_size} bytes") - ) - } -} - -#[cfg(target_arch = "loongarch64")] -fn fill_vm_region(load_addr: GuestPhysAddr, size: usize, byte: u8, vm: AxVMRef) -> AxResult { - let image_load_regions = vm.get_image_load_region(load_addr, size)?; - let mut filled_size = 0; - - for region in image_load_regions { - unsafe { - core::ptr::write_bytes(region.as_mut_ptr(), byte, region.len()); - } - crate::clean_dcache_range((region.as_ptr() as usize).into(), region.len()); - filled_size += region.len(); - } - - if filled_size == size { - Ok(()) - } else { - ax_err!( - InvalidData, - format!("VM memory was only partially filled: {filled_size}/{size} bytes") - ) - } -} - -#[cfg(any(feature = "fs", feature = "host-fs"))] -pub mod fs { - #[cfg(target_arch = "loongarch64")] - use alloc::boxed::Box; - use alloc::vec::Vec; - - use ax_errno::{AxResult, ax_err, ax_err_type}; - - use super::*; - - pub fn kernel_read( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, - read_size: usize, - ) -> AxResult> { - let file_name = &config.kernel.kernel_path; - provider.read_file_exact(file_name, read_size) - } - - /// Loads the VM image files from the filesystem - /// into the guest VM's memory space based on the VM configuration. - pub(crate) fn load_vm_images_from_filesystem(loader: &mut ImageLoader) -> AxResult { - info!("Loading VM images from filesystem"); - #[cfg(target_arch = "x86_64")] - { - if should_direct_boot_x86_linux(&loader.config) { - let kernel_probe = - kernel_read(&loader.config, loader.provider, x86_linux::HEADER_READ_SIZE); - match kernel_probe { - Ok(data) => { - if let Some(header) = detect_x86_linux_image(&data) { - let kernel = read_image_file( - &loader.config.kernel.kernel_path, - loader.provider, - )?; - return loader.load_x86_linux_images_from_filesystem(header, &kernel); - } - } - Err(err) => debug!("Unable to probe x86 Linux bzImage header: {err:?}"), - } - } - } - // Load kernel image. - #[cfg(target_arch = "loongarch64")] - if loader.loongarch_uefi_boot() { - loader.load_loongarch_uefi_firmware_dtb()?; - loader.add_loongarch_uefi_fw_cfg_from_filesystem()?; - loader.load_loongarch_uefi_firmware_from_filesystem()?; - return Ok(()); - } else { - load_vm_image( - &loader.config.kernel.kernel_path, - loader.kernel_load_gpa, - loader.vm.clone(), - loader.provider, - )?; - } - #[cfg(not(target_arch = "loongarch64"))] - load_vm_image( - &loader.config.kernel.kernel_path, - loader.kernel_load_gpa, - loader.vm.clone(), - loader.provider, - )?; - // Load boot firmware image if needed. - if loader.config.kernel.enable_bios - && let Some(bios_path) = loader.config.kernel.boot_firmware_path() - { - if let Some(bios_load_addr) = loader.bios_load_gpa { - #[cfg(target_arch = "x86_64")] - { - if should_patch_x86_multiboot_info(&loader.config) { - let bios_image = read_image_file(bios_path, loader.provider)?; - validate_x86_bios_patch_region(&bios_image)?; - load_vm_image_from_memory(&bios_image, bios_load_addr, loader.vm.clone())?; - loader.load_x86_multiboot_info(&bios_image, bios_load_addr)?; - } else { - load_vm_image( - bios_path, - bios_load_addr, - loader.vm.clone(), - loader.provider, - )?; - } - } - #[cfg(not(target_arch = "x86_64"))] - load_vm_image( - bios_path, - bios_load_addr, - loader.vm.clone(), - loader.provider, - )?; - } else { - return ax_err!(NotFound, "boot firmware load addr is missed"); - } - }; - #[cfg(target_arch = "x86_64")] - if loader.should_load_default_x86_boot_image() { - let bios_load_gpa = builtin_x86_bios_load_gpa(loader.bios_load_gpa)?; - info!( - "Loading built-in x86 boot image at GPA {:#x}", - bios_load_gpa.as_usize() - ); - load_vm_image_from_memory( - x86_boot::DEFAULT_BIOS_IMAGE, - bios_load_gpa, - loader.vm.clone(), - )?; - #[cfg(target_arch = "x86_64")] - loader.load_x86_multiboot_info(x86_boot::DEFAULT_BIOS_IMAGE, bios_load_gpa)?; - } - - // Load Ramdisk image if needed. - if let Some(ramdisk_path) = &loader.config.kernel.ramdisk_path { - loader.load_ramdisk_from_filesystem(ramdisk_path)?; - }; - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - { - // Load DTB image if needed. - if let Some(dtb) = &loader.guest_dtb { - loader.load_guest_dtb(dtb)?; - } - } - - Ok(()) - } - - #[cfg(target_arch = "loongarch64")] - impl ImageLoader<'_> { - fn add_loongarch_uefi_fw_cfg_from_filesystem(&self) -> AxResult { - let kernel = read_full_image(&self.config.kernel.kernel_path, self.provider)?; - let kernel: &'static [u8] = Box::leak(kernel.into_boxed_slice()); - let ramdisk = if let Some(path) = &self.config.kernel.ramdisk_path { - let ramdisk = read_full_image(path, self.provider)?; - Some(Box::leak(ramdisk.into_boxed_slice()) as &'static [u8]) - } else { - None - }; - - debug!( - "VM[{}] configuring LoongArch UEFI fw_cfg payloads from filesystem: kernel={} \ - bytes, ramdisk={:?}", - self.config.base.id, - kernel.len(), - ramdisk.map(|data| data.len()) - ); - self.add_loongarch_uefi_fw_cfg(kernel, ramdisk) - } - - fn load_loongarch_uefi_firmware_from_filesystem(&self) -> AxResult { - let firmware = - provider_firmware_image_for_vm(&self.config, self.provider).ok_or_else(|| { - ax_err_type!( - NotFound, - "LoongArch UEFI boot requires a build-time firmware image" - ) - })?; - self.load_loongarch_uefi_firmware_image(firmware) - } - } - - #[cfg(target_arch = "x86_64")] - impl ImageLoader<'_> { - fn load_x86_linux_images_from_filesystem( - &mut self, - header: x86_linux::X86LinuxHeader, - kernel: &[u8], - ) -> AxResult { - self.adjust_x86_linux_dma_identity_layout()?; - let payload = x86_linux_payload(&header, kernel)?; - let initrd = if let Some(ramdisk_path) = &self.config.kernel.ramdisk_path { - let ramdisk_size = image_size(ramdisk_path, self.provider)?; - Some(x86_linux::X86LinuxRange::new( - self.ramdisk_load_gpa()?.as_usize(), - ramdisk_size, - )) - } else { - None - }; - let layout = x86_linux::X86LinuxLoadLayout::new( - &header, - self.kernel_load_gpa.as_usize(), - payload.len(), - initrd, - ) - .map_err(x86_linux_layout_error)?; - - self.load_x86_linux_layout(header, layout, kernel)?; - load_vm_image_from_memory(payload, self.kernel_load_gpa, self.vm.clone())?; - - if let Some(ramdisk_path) = &self.config.kernel.ramdisk_path { - self.load_ramdisk_from_filesystem(ramdisk_path)?; - } - - Ok(()) - } - } - - pub(crate) fn load_vm_image( - image_path: &str, - image_load_gpa: GuestPhysAddr, - vm: AxVMRef, - provider: &dyn BootImageProvider, - ) -> AxResult { - let image = provider.read_file(image_path)?; - let image_size = image.len(); - - let image_load_regions = vm.get_image_load_region(image_load_gpa, image_size)?; - let mut offset = 0; - - for buffer in image_load_regions { - let end = offset + buffer.len(); - let data = image.get(offset..end).ok_or_else(|| { - ax_err_type!( - InvalidData, - format!("Image {} has an invalid load region layout", image_path) - ) - })?; - buffer.copy_from_slice(data); - offset = end; - - crate::clean_dcache_range((buffer.as_ptr() as usize).into(), buffer.len()); - } - - Ok(()) - } - - #[cfg(target_arch = "x86_64")] - fn read_image_file(image_path: &str, provider: &dyn BootImageProvider) -> AxResult> { - provider.read_file(image_path) - } - - pub fn image_size(file_name: &str, provider: &dyn BootImageProvider) -> AxResult { - provider.file_size(file_name) - } - - #[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" - ))] - pub fn read_full_image(file_name: &str, provider: &dyn BootImageProvider) -> AxResult> { - provider.read_file(file_name) - } -} - -#[cfg(target_arch = "x86_64")] -fn should_patch_x86_multiboot_info(config: &AxVMCrateConfig) -> bool { - config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot -} - -#[cfg(target_arch = "x86_64")] -fn should_direct_boot_x86_linux(config: &AxVMCrateConfig) -> bool { - !config.kernel.enable_bios && config.kernel.effective_boot_protocol() == VMBootProtocol::Direct -} - -#[cfg(target_arch = "x86_64")] -fn detect_x86_linux_image(image: &[u8]) -> Option { - match x86_linux::X86LinuxHeader::parse(image) { - Ok(header) => Some(header), - Err(err) => { - debug!("Not an x86 Linux bzImage: {err:?}"); - None - } - } -} - -#[cfg(target_arch = "x86_64")] -fn x86_linux_payload<'a>( - header: &x86_linux::X86LinuxHeader, - image: &'a [u8], -) -> AxResult<&'a [u8]> { - let payload_offset = header.payload_offset(); - image.get(payload_offset..).ok_or_else(|| { - ax_errno::ax_err_type!( - InvalidInput, - format!( - "x86 Linux bzImage payload offset {:#x} exceeds image size {:#x}", - payload_offset, - image.len() - ) - ) - }) -} - -#[cfg(target_arch = "x86_64")] -fn x86_linux_layout_error(err: x86_linux::X86LinuxLayoutError) -> ax_errno::AxError { - ax_errno::ax_err_type!( - InvalidInput, - format!("invalid x86 Linux memory layout: {err:?}") - ) -} - -#[cfg(target_arch = "x86_64")] -fn builtin_x86_bios_load_gpa(configured_gpa: Option) -> AxResult { - let default_gpa = GuestPhysAddr::from(x86_boot::DEFAULT_BIOS_LOAD_GPA); - match configured_gpa { - Some(gpa) if gpa != default_gpa => Err(ax_errno::ax_err_type!( - InvalidInput, - format!( - "built-in x86 BIOS must be loaded at GPA {:#x}, but bios_load_addr is {:#x}; set \ - bios_path to use a relocatable external BIOS image", - default_gpa.as_usize(), - gpa.as_usize() - ) - )), - Some(gpa) => Ok(gpa), - None => Ok(default_gpa), - } -} - -#[cfg(target_arch = "x86_64")] -fn validate_x86_bios_patch_region(bios_image: &[u8]) -> AxResult { - let patch_end = x86_boot::AXVM_BIOS_EBX_IMM_OFFSET + core::mem::size_of::(); - if bios_image.len() < patch_end { - return Err(ax_errno::ax_err_type!( - InvalidInput, - format!( - "x86 BIOS image is too small for multiboot info patch: size {}, need at least {} \ - bytes for EBX immediate at offset {:#x}", - bios_image.len(), - patch_end, - x86_boot::AXVM_BIOS_EBX_IMM_OFFSET - ) - )); - } - - if bios_image[x86_boot::AXVM_BIOS_EBX_IMM_OFFSET - 1] != x86_boot::MOV_EBX_IMM32_OPCODE { - return Err(ax_errno::ax_err_type!( - InvalidInput, - format!( - "x86 BIOS image does not match axvm-bios layout: expected mov ebx, imm32 opcode \ - at offset {:#x}", - x86_boot::AXVM_BIOS_EBX_IMM_OFFSET - 1 - ) - )); - } - - Ok(()) -} - -#[cfg(target_arch = "x86_64")] -fn write_u32(buffer: &mut [u8], offset: usize, value: u32) { - buffer[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); -} - -#[cfg(target_arch = "x86_64")] -fn write_u64(buffer: &mut [u8], offset: usize, value: u64) { - buffer[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); -} - -#[cfg(all(test, target_arch = "x86_64"))] -mod tests { - use super::*; - - #[test] - fn built_in_x86_bios_uses_default_gpa_when_unspecified() { - assert_eq!( - builtin_x86_bios_load_gpa(None).unwrap(), - GuestPhysAddr::from(x86_boot::DEFAULT_BIOS_LOAD_GPA) - ); - } - - #[test] - fn built_in_x86_bios_accepts_explicit_default_gpa() { - let default_gpa = GuestPhysAddr::from(x86_boot::DEFAULT_BIOS_LOAD_GPA); - - assert_eq!( - builtin_x86_bios_load_gpa(Some(default_gpa)).unwrap(), - default_gpa - ); - } - - #[test] - fn built_in_x86_bios_rejects_non_default_gpa() { - let invalid_gpa = GuestPhysAddr::from(x86_boot::DEFAULT_BIOS_LOAD_GPA + 0x1000); - - assert!(builtin_x86_bios_load_gpa(Some(invalid_gpa)).is_err()); - } - - #[test] - fn legacy_x86_bios_config_uses_multiboot_patch() { - let mut cfg = AxVMCrateConfig::default(); - cfg.kernel.enable_bios = true; - - assert!(should_patch_x86_multiboot_info(&cfg)); - } - - #[test] - fn x86_uefi_config_skips_multiboot_patch() { - let mut cfg = AxVMCrateConfig::default(); - cfg.kernel.enable_bios = true; - cfg.kernel.boot_protocol = Some(VMBootProtocol::Uefi); - - assert!(!should_patch_x86_multiboot_info(&cfg)); - } - - #[test] - fn x86_linux_direct_boot_requires_direct_protocol() { - let mut cfg = AxVMCrateConfig::default(); - - assert!(should_direct_boot_x86_linux(&cfg)); - - cfg.kernel.enable_bios = true; - assert!(!should_direct_boot_x86_linux(&cfg)); - - cfg.kernel.boot_protocol = Some(VMBootProtocol::Uefi); - assert!(!should_direct_boot_x86_linux(&cfg)); - - cfg.kernel.boot_protocol = Some(VMBootProtocol::Direct); - assert!(!should_direct_boot_x86_linux(&cfg)); - - cfg.kernel.enable_bios = false; - assert!(should_direct_boot_x86_linux(&cfg)); - } -} diff --git a/virtualization/axvm/src/arch/x86_64/boot/images/x86/mod.rs b/virtualization/axvm/src/arch/x86_64/boot/images/x86/mod.rs deleted file mode 100644 index 5aa554161c..0000000000 --- a/virtualization/axvm/src/arch/x86_64/boot/images/x86/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! x86 image loading helpers. - -pub mod linux; -pub mod linux_boot; -pub mod mptable; -pub mod multiboot; - -pub mod boot_params; diff --git a/virtualization/axvm/src/arch/x86_64/boot/images/x86/linux.rs b/virtualization/axvm/src/arch/x86_64/boot/linux.rs similarity index 100% rename from virtualization/axvm/src/arch/x86_64/boot/images/x86/linux.rs rename to virtualization/axvm/src/arch/x86_64/boot/linux.rs diff --git a/virtualization/axvm/src/arch/x86_64/boot/images/x86/linux_boot.rs b/virtualization/axvm/src/arch/x86_64/boot/linux_boot.rs similarity index 98% rename from virtualization/axvm/src/arch/x86_64/boot/images/x86/linux_boot.rs rename to virtualization/axvm/src/arch/x86_64/boot/linux_boot.rs index 18fb82ab43..b0a8d2fac6 100644 --- a/virtualization/axvm/src/arch/x86_64/boot/images/x86/linux_boot.rs +++ b/virtualization/axvm/src/arch/x86_64/boot/linux_boot.rs @@ -103,8 +103,10 @@ fn write_u32(buffer: &mut [u8], offset: usize, value: u32) { #[cfg(test)] mod tests { - use super::*; - use crate::boot::images::x86::linux::{BOOT_PARAMS_GPA, X86LinuxHeader, X86LinuxRange}; + use super::{ + super::linux::{BOOT_PARAMS_GPA, X86LinuxHeader, X86LinuxRange}, + *, + }; const SETUP_SECTS_OFFSET: usize = 0x1f1; const BOOT_FLAG_OFFSET: usize = 0x1fe; diff --git a/virtualization/axvm/src/arch/x86_64/boot/mod.rs b/virtualization/axvm/src/arch/x86_64/boot/mod.rs new file mode 100644 index 0000000000..cd6bc5d8b2 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/boot/mod.rs @@ -0,0 +1,537 @@ +//! x86_64 Linux, BIOS, UEFI, and MP-table image planning. + +use alloc::format; + +use ax_errno::{AxResult, ax_err_type}; +use axvm_types::GuestPhysAddr; +use axvmconfig::{EmulatedDeviceType, VMBootProtocol, VmMemMappingType}; + +use super::X86_64Arch; +use crate::{ + architecture::BootImagePlatform, + boot::{ + BootImageProvider, StaticVmImage, + images::{ImageLoaderCore, load_vm_image_from_memory}, + }, +}; + +mod boot_params; +mod linux; +mod linux_boot; +mod mptable; +mod multiboot; + +pub struct ImageLoader<'a>(ImageLoaderCore<'a>); + +impl<'a> ImageLoader<'a> { + pub fn new( + main_memory: crate::VMMemoryRegion, + config: axvmconfig::AxVMCrateConfig, + vm: crate::AxVMRef, + provider: &'a dyn BootImageProvider, + ) -> Self { + Self(ImageLoaderCore::new( + main_memory, + config, + vm, + provider, + None, + )) + } + + pub fn load(&mut self) -> AxResult { + self.0.load() + } +} + +impl BootImagePlatform for X86_64Arch { + fn load_images_from_memory( + loader: &mut ImageLoaderCore<'_>, + images: StaticVmImage, + ) -> AxResult { + if should_direct_boot_linux(&loader.config) + && let Some(header) = detect_linux_image(images.kernel) + { + return load_linux_from_memory(loader, header, images.kernel, images.ramdisk); + } + + load_vm_image_from_memory(images.kernel, loader.kernel_load_gpa, loader.vm.clone())?; + if let Some(ramdisk) = images.ramdisk { + loader.load_ramdisk_from_memory(ramdisk)?; + } + load_boot_image_from_memory(loader, images.bios) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + fn load_images_from_filesystem(loader: &mut ImageLoaderCore<'_>) -> AxResult { + if should_direct_boot_linux(&loader.config) { + let probe = crate::boot::images::fs::kernel_read( + &loader.config, + loader.provider, + linux::HEADER_READ_SIZE, + ); + if let Ok(data) = probe + && let Some(header) = detect_linux_image(&data) + { + let kernel = crate::boot::images::fs::read_full_image( + &loader.config.kernel.kernel_path, + loader.provider, + )?; + return load_linux_from_filesystem(loader, header, &kernel); + } + } + + crate::boot::images::fs::load_vm_image( + &loader.config.kernel.kernel_path, + loader.kernel_load_gpa, + loader.vm.clone(), + loader.provider, + )?; + load_boot_image_from_filesystem(loader)?; + if let Some(ramdisk_path) = &loader.config.kernel.ramdisk_path { + loader.load_ramdisk_from_filesystem(ramdisk_path)?; + } + Ok(()) + } + + fn is_x86_linux_image_config( + config: &axvmconfig::AxVMCrateConfig, + provider: &dyn BootImageProvider, + ) -> bool { + if !should_direct_boot_linux(config) { + return false; + } + match config.kernel.image_location.as_deref() { + Some("memory") => provider + .static_vm_images() + .iter() + .find(|image| image.id == config.base.id) + .and_then(|image| detect_linux_image(image.kernel)) + .is_some(), + #[cfg(any(feature = "fs", feature = "host-fs"))] + Some("fs") => { + crate::boot::images::fs::kernel_read(config, provider, linux::HEADER_READ_SIZE) + .ok() + .and_then(|image| detect_linux_image(&image)) + .is_some() + } + _ => false, + } + } +} + +fn load_linux_from_memory( + loader: &mut ImageLoaderCore<'_>, + header: linux::X86LinuxHeader, + kernel: &[u8], + ramdisk: Option<&[u8]>, +) -> AxResult { + adjust_linux_dma_identity_layout(loader); + let payload = linux_payload(&header, kernel)?; + let initrd = ramdisk + .map(|image| { + loader + .ramdisk_load_gpa() + .map(|gpa| linux::X86LinuxRange::new(gpa.as_usize(), image.len())) + }) + .transpose()?; + let layout = linux::X86LinuxLoadLayout::new( + &header, + loader.kernel_load_gpa.as_usize(), + payload.len(), + initrd, + ) + .map_err(linux_layout_error)?; + + load_linux_layout(loader, header, layout, kernel)?; + load_vm_image_from_memory(payload, loader.kernel_load_gpa, loader.vm.clone())?; + if let Some(ramdisk) = ramdisk { + loader.load_ramdisk_from_memory(ramdisk)?; + } + Ok(()) +} + +#[cfg(any(feature = "fs", feature = "host-fs"))] +fn load_linux_from_filesystem( + loader: &mut ImageLoaderCore<'_>, + header: linux::X86LinuxHeader, + kernel: &[u8], +) -> AxResult { + adjust_linux_dma_identity_layout(loader); + let payload = linux_payload(&header, kernel)?; + let initrd = loader + .config + .kernel + .ramdisk_path + .as_deref() + .map(|path| -> AxResult<_> { + let size = crate::boot::images::fs::image_size(path, loader.provider)?; + Ok(linux::X86LinuxRange::new( + loader.ramdisk_load_gpa()?.as_usize(), + size, + )) + }) + .transpose()?; + let layout = linux::X86LinuxLoadLayout::new( + &header, + loader.kernel_load_gpa.as_usize(), + payload.len(), + initrd, + ) + .map_err(linux_layout_error)?; + + load_linux_layout(loader, header, layout, kernel)?; + load_vm_image_from_memory(payload, loader.kernel_load_gpa, loader.vm.clone())?; + if let Some(path) = &loader.config.kernel.ramdisk_path { + loader.load_ramdisk_from_filesystem(path)?; + } + Ok(()) +} + +fn load_boot_image_from_memory(loader: &ImageLoaderCore<'_>, bios: Option<&[u8]>) -> AxResult { + if !loader.config.kernel.enable_bios { + return Ok(()); + } + if let Some(bios) = bios { + let load_gpa = loader + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; + load_vm_image_from_memory(bios, load_gpa, loader.vm.clone())?; + if should_patch_multiboot_info(&loader.config) { + load_multiboot_info(loader, bios, load_gpa)?; + } + return Ok(()); + } + + if loader.config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { + return load_uefi_from_configured_path(loader); + } + if should_load_default_boot_image(loader) { + let load_gpa = builtin_bios_load_gpa(loader.bios_load_gpa)?; + load_vm_image_from_memory(multiboot::DEFAULT_BIOS_IMAGE, load_gpa, loader.vm.clone())?; + load_multiboot_info(loader, multiboot::DEFAULT_BIOS_IMAGE, load_gpa)?; + } + Ok(()) +} + +#[cfg(any(feature = "fs", feature = "host-fs"))] +fn load_boot_image_from_filesystem(loader: &ImageLoaderCore<'_>) -> AxResult { + if !loader.config.kernel.enable_bios { + return Ok(()); + } + if let Some(path) = loader.config.kernel.boot_firmware_path() { + let load_gpa = loader + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; + if should_patch_multiboot_info(&loader.config) { + let bios = crate::boot::images::fs::read_full_image(path, loader.provider)?; + validate_bios_patch_region(&bios)?; + load_vm_image_from_memory(&bios, load_gpa, loader.vm.clone())?; + load_multiboot_info(loader, &bios, load_gpa) + } else { + crate::boot::images::fs::load_vm_image( + path, + load_gpa, + loader.vm.clone(), + loader.provider, + ) + } + } else if should_load_default_boot_image(loader) { + let load_gpa = builtin_bios_load_gpa(loader.bios_load_gpa)?; + load_vm_image_from_memory(multiboot::DEFAULT_BIOS_IMAGE, load_gpa, loader.vm.clone())?; + load_multiboot_info(loader, multiboot::DEFAULT_BIOS_IMAGE, load_gpa) + } else { + Ok(()) + } +} + +fn load_uefi_from_configured_path(loader: &ImageLoaderCore<'_>) -> AxResult { + let path = loader + .config + .kernel + .boot_firmware_path() + .ok_or_else(|| ax_err_type!(NotFound, "UEFI firmware image path is missed"))?; + let load_gpa = loader + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "UEFI firmware load addr is missed"))?; + #[cfg(any(feature = "fs", feature = "host-fs"))] + { + crate::boot::images::fs::load_vm_image(path, load_gpa, loader.vm.clone(), loader.provider) + } + #[cfg(not(any(feature = "fs", feature = "host-fs")))] + { + let _ = (path, load_gpa); + ax_errno::ax_err!( + Unsupported, + "UEFI firmware path requires the fs feature when no firmware image buffer is available" + ) + } +} + +fn adjust_linux_dma_identity_layout(loader: &mut ImageLoaderCore<'_>) { + if !loader.main_memory.is_identical() { + return; + } + let memory_base = loader.main_memory.gpa.as_usize(); + loader.kernel_load_gpa = + GuestPhysAddr::from(memory_base + loader.config.kernel.kernel_load_addr); + if let Some(ramdisk_load_addr) = loader.config.kernel.ramdisk_load_addr { + loader.ramdisk_load_gpa = Some(GuestPhysAddr::from(memory_base + ramdisk_load_addr)); + } + loader.vm.with_config(|config| { + config.image_config.kernel_load_gpa = loader.kernel_load_gpa; + if let Some(load_gpa) = loader.ramdisk_load_gpa + && let Some(ramdisk) = config.image_config.ramdisk.as_mut() + { + ramdisk.load_gpa = load_gpa; + } + }); +} + +fn load_linux_layout( + loader: &ImageLoaderCore<'_>, + header: linux::X86LinuxHeader, + layout: linux::X86LinuxLoadLayout, + kernel: &[u8], +) -> AxResult { + let boot_params = build_boot_params(loader, header, layout, kernel)?; + let boot_stub = linux_boot::build_boot_image(&layout).map_err(|err| { + ax_err_type!( + InvalidInput, + format!("failed to build x86 Linux boot stub: {err:?}") + ) + })?; + load_vm_image_from_memory( + &boot_params, + layout.boot_params.start.into(), + loader.vm.clone(), + )?; + load_vm_image_from_memory(&boot_stub, layout.boot_stub.start.into(), loader.vm.clone())?; + load_vm_image_from_memory( + &mptable::build(), + mptable::MP_TABLE_GPA.into(), + loader.vm.clone(), + )?; + let entry = GuestPhysAddr::from(linux_boot::DEFAULT_LINUX_BOOT_LOAD_GPA); + loader.vm.with_config(|config| { + config.cpu_config.bsp_entry = entry; + config.cpu_config.ap_entry = entry; + }); + Ok(()) +} + +fn build_boot_params( + loader: &ImageLoaderCore<'_>, + header: linux::X86LinuxHeader, + layout: linux::X86LinuxLoadLayout, + kernel: &[u8], +) -> AxResult<[u8; linux::BOOT_PARAMS_SIZE]> { + let mut builder = boot_params::BootParamsBuilder::new( + kernel, + header, + layout, + linux::X86LinuxRange::new(loader.main_memory.gpa.as_usize(), loader.main_memory.size()), + ); + let command_line = loader.config.kernel.cmdline.as_deref().ok_or_else(|| { + ax_err_type!( + InvalidInput, + "x86 Linux direct boot requires kernel.cmdline in the VM config" + ) + })?; + builder.set_command_line(command_line).map_err(|err| { + ax_err_type!( + InvalidInput, + format!("invalid x86 Linux command line: {err:?}") + ) + })?; + for memory in &loader.config.kernel.memory_regions { + if memory.map_type == VmMemMappingType::MapAlloc { + builder.add_ram_range(linux::X86LinuxRange::new(memory.gpa, memory.size)); + } + } + for device in &loader.config.devices.passthrough_devices { + builder.add_reserved_range(linux::X86LinuxRange::new(device.base_gpa, device.length)); + } + for address in &loader.config.devices.passthrough_addresses { + builder.add_reserved_range(linux::X86LinuxRange::new(address.base_gpa, address.length)); + } + for device in &loader.config.devices.emu_devices { + if matches!(device.emu_type, EmulatedDeviceType::X86IoApic) { + builder.add_reserved_range(linux::X86LinuxRange::new(device.base_gpa, device.length)); + } + } + builder.add_reserved_range(mptable::reserved_range()); + builder.build().map_err(|err| { + ax_err_type!( + InvalidInput, + format!("failed to build x86 boot_params: {err:?}") + ) + }) +} + +fn load_multiboot_info( + loader: &ImageLoaderCore<'_>, + bios_image: &[u8], + bios_load_gpa: GuestPhysAddr, +) -> AxResult { + const INFO_GPA: usize = 0x6000; + const MMAP_GPA: usize = 0x6040; + let mem_base = loader.main_memory.gpa.as_usize() as u64; + let mem_size = loader.main_memory.size() as u64; + let mut info = [0u8; 52]; + write_u32(&mut info, 0, (1 << 0) | (1 << 6)); + write_u32(&mut info, 4, 639); + write_u32( + &mut info, + 8, + (mem_size.saturating_sub(0x100000) / 1024) as u32, + ); + write_u32(&mut info, 44, 24); + write_u32(&mut info, 48, MMAP_GPA as u32); + let mut mmap = [0u8; 24]; + write_u32(&mut mmap, 0, 20); + write_u64(&mut mmap, 4, mem_base); + write_u64(&mut mmap, 12, mem_size); + write_u32(&mut mmap, 20, 1); + validate_bios_patch_region(bios_image)?; + load_vm_image_from_memory(&info, INFO_GPA.into(), loader.vm.clone())?; + load_vm_image_from_memory(&mmap, MMAP_GPA.into(), loader.vm.clone())?; + load_vm_image_from_memory( + &(INFO_GPA as u32).to_le_bytes(), + (bios_load_gpa.as_usize() + multiboot::AXVM_BIOS_EBX_IMM_OFFSET).into(), + loader.vm.clone(), + ) +} + +fn should_direct_boot_linux(config: &axvmconfig::AxVMCrateConfig) -> bool { + !config.kernel.enable_bios && config.kernel.effective_boot_protocol() == VMBootProtocol::Direct +} + +fn should_patch_multiboot_info(config: &axvmconfig::AxVMCrateConfig) -> bool { + config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot +} + +fn should_load_default_boot_image(loader: &ImageLoaderCore<'_>) -> bool { + loader.config.kernel.enable_bios + && loader.config.kernel.boot_firmware_path().is_none() + && loader.config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot +} + +fn detect_linux_image(image: &[u8]) -> Option { + linux::X86LinuxHeader::parse(image).ok() +} + +fn linux_payload<'a>(header: &linux::X86LinuxHeader, image: &'a [u8]) -> AxResult<&'a [u8]> { + image.get(header.payload_offset()..).ok_or_else(|| { + ax_err_type!( + InvalidInput, + format!( + "x86 Linux bzImage payload offset {:#x} exceeds image size {:#x}", + header.payload_offset(), + image.len() + ) + ) + }) +} + +fn linux_layout_error(err: linux::X86LinuxLayoutError) -> ax_errno::AxError { + ax_err_type!( + InvalidInput, + format!("invalid x86 Linux memory layout: {err:?}") + ) +} + +fn builtin_bios_load_gpa(configured: Option) -> AxResult { + let default = GuestPhysAddr::from(multiboot::DEFAULT_BIOS_LOAD_GPA); + match configured { + Some(gpa) if gpa != default => Err(ax_err_type!( + InvalidInput, + format!( + "built-in x86 BIOS must be loaded at GPA {:#x}, but bios_load_addr is {:#x}", + default.as_usize(), + gpa.as_usize() + ) + )), + Some(gpa) => Ok(gpa), + None => Ok(default), + } +} + +fn validate_bios_patch_region(bios: &[u8]) -> AxResult { + let patch_end = multiboot::AXVM_BIOS_EBX_IMM_OFFSET + core::mem::size_of::(); + if bios.len() < patch_end + || bios[multiboot::AXVM_BIOS_EBX_IMM_OFFSET - 1] != multiboot::MOV_EBX_IMM32_OPCODE + { + return Err(ax_err_type!( + InvalidInput, + "x86 BIOS image does not match the AxVM multiboot patch layout" + )); + } + Ok(()) +} + +fn write_u32(buffer: &mut [u8], offset: usize, value: u32) { + buffer[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn write_u64(buffer: &mut [u8], offset: usize, value: u64) { + buffer[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn built_in_bios_uses_default_gpa_when_unspecified() { + assert_eq!( + builtin_bios_load_gpa(None).unwrap(), + GuestPhysAddr::from(multiboot::DEFAULT_BIOS_LOAD_GPA) + ); + } + + #[test] + fn built_in_bios_accepts_explicit_default_gpa() { + let default = GuestPhysAddr::from(multiboot::DEFAULT_BIOS_LOAD_GPA); + assert_eq!(builtin_bios_load_gpa(Some(default)).unwrap(), default); + } + + #[test] + fn built_in_bios_rejects_non_default_gpa() { + let invalid = GuestPhysAddr::from(multiboot::DEFAULT_BIOS_LOAD_GPA + 0x1000); + assert!(builtin_bios_load_gpa(Some(invalid)).is_err()); + } + + #[test] + fn legacy_bios_config_uses_multiboot_patch() { + let mut config = axvmconfig::AxVMCrateConfig::default(); + config.kernel.enable_bios = true; + assert!(should_patch_multiboot_info(&config)); + } + + #[test] + fn uefi_config_skips_multiboot_patch() { + let mut config = axvmconfig::AxVMCrateConfig::default(); + config.kernel.enable_bios = true; + config.kernel.boot_protocol = Some(VMBootProtocol::Uefi); + assert!(!should_patch_multiboot_info(&config)); + } + + #[test] + fn linux_direct_boot_requires_direct_protocol_without_bios() { + let mut config = axvmconfig::AxVMCrateConfig::default(); + assert!(should_direct_boot_linux(&config)); + + config.kernel.enable_bios = true; + assert!(!should_direct_boot_linux(&config)); + + config.kernel.boot_protocol = Some(VMBootProtocol::Uefi); + assert!(!should_direct_boot_linux(&config)); + + config.kernel.boot_protocol = Some(VMBootProtocol::Direct); + assert!(!should_direct_boot_linux(&config)); + + config.kernel.enable_bios = false; + assert!(should_direct_boot_linux(&config)); + } +} diff --git a/virtualization/axvm/src/arch/x86_64/boot/images/x86/mptable.rs b/virtualization/axvm/src/arch/x86_64/boot/mptable.rs similarity index 92% rename from virtualization/axvm/src/arch/x86_64/boot/images/x86/mptable.rs rename to virtualization/axvm/src/arch/x86_64/boot/mptable.rs index 1c228d22e4..5a134dd123 100644 --- a/virtualization/axvm/src/arch/x86_64/boot/images/x86/mptable.rs +++ b/virtualization/axvm/src/arch/x86_64/boot/mptable.rs @@ -22,10 +22,6 @@ const IO_APIC_VERSION: u8 = 0x11; const BUS_ID_PCI: u8 = 0; const BUS_ID_ISA: u8 = 1; -pub const QEMU_PASSTHROUGH_BLOCK_DEVICE: u8 = 3; -pub const QEMU_PASSTHROUGH_BLOCK_FUNCTION: u8 = 0; -pub const QEMU_PASSTHROUGH_BLOCK_PIN: u8 = 1; - const MP_IRQ_FLAGS_CONFORMING: u16 = 0; const MP_IRQ_FLAGS_ACTIVE_LOW: u16 = 0x3; const MP_IRQ_FLAGS_LEVEL_TRIGGERED: u16 = 0xc; @@ -169,13 +165,6 @@ const fn pci_intx_gsi(dev: u8, pin: u8) -> u8 { 16 + ((dev + pin) & 3) } -pub const fn qemu_passthrough_block_gsi() -> usize { - pci_intx_gsi( - QEMU_PASSTHROUGH_BLOCK_DEVICE, - QEMU_PASSTHROUGH_BLOCK_PIN - 1, - ) as usize -} - fn interrupt_entry( interrupt_type: u8, flags: u16, @@ -225,19 +214,20 @@ mod tests { #[test] fn pci_intx_entries_are_low_active_level_triggered() { - let source_irq = (QEMU_PASSTHROUGH_BLOCK_DEVICE << 2) | (QEMU_PASSTHROUGH_BLOCK_PIN - 1); + let (device, _, pin, guest_gsi) = crate::boot::x86_qemu_passthrough_block_intx(); + let source_irq = (device << 2) | (pin - 1); let entry = interrupt_entry( 0, PCI_INTX_IRQ_FLAGS, BUS_ID_PCI, source_irq, - qemu_passthrough_block_gsi() as u8, + guest_gsi as u8, ); assert_eq!(u16::from_le_bytes([entry[2], entry[3]]), 0x0f); assert_eq!(entry[4], BUS_ID_PCI); assert_eq!(entry[5], source_irq); - assert_eq!(entry[7], qemu_passthrough_block_gsi() as u8); + assert_eq!(entry[7], guest_gsi as u8); } #[test] diff --git a/virtualization/axvm/src/arch/x86_64/boot/images/x86/multiboot.rs b/virtualization/axvm/src/arch/x86_64/boot/multiboot.rs similarity index 100% rename from virtualization/axvm/src/arch/x86_64/boot/images/x86/multiboot.rs rename to virtualization/axvm/src/arch/x86_64/boot/multiboot.rs diff --git a/virtualization/axvm/src/arch/x86_64/capabilities.rs b/virtualization/axvm/src/arch/x86_64/capabilities.rs new file mode 100644 index 0000000000..156df7c188 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/capabilities.rs @@ -0,0 +1,64 @@ +//! x86_64 implementations of AxVM platform capability hooks. + +use alloc::{format, sync::Arc}; + +use ax_errno::{AxResult, ax_err_type}; +use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; + +use super::X86_64Arch; +use crate::architecture::{AddressSpacePlatform, DevicePlatform, HostTimePlatform}; + +impl DevicePlatform for X86_64Arch { + fn register_devices( + _vm: &crate::AxVM, + config: &crate::config::AxVMConfig, + devices: &mut axdevice::AxVmDevices, + ) -> AxResult { + for port in config.pass_through_ports() { + let passthrough = Arc::new(super::port::HostPortPassthrough::new( + port.base, + port.length, + )?); + let range = passthrough.address_range(); + debug!( + "PT port region: [{:#x}~{:#x}]", + range.start.number(), + range.end.number(), + ); + devices + .register(PortDeviceAdapter::from_arc(passthrough)) + .map_err(|err| ax_err_type!(InvalidInput, format!("register PT port: {err:?}")))?; + } + for config in config.emu_devices() { + super::register_arch_device(config, devices)?; + } + Ok(()) + } +} + +impl AddressSpacePlatform for X86_64Arch { + #[cfg(feature = "vmx")] + fn append_owned_regions(regions: &mut alloc::vec::Vec) { + regions.push(crate::layout::GuestOwnedRegion::new( + x86_vcpu::X86_APIC_ACCESS_GPA, + ax_memory_addr::PAGE_SIZE_4K, + crate::layout::VmRegionKind::Reserved, + )); + } + + #[cfg(feature = "vmx")] + fn map_address_space( + address_space: &mut axaddrspace::AddrSpace, + ) -> AxResult { + address_space.map_linear( + axvm_types::GuestPhysAddr::from(x86_vcpu::X86_APIC_ACCESS_GPA), + super::x86_apic_access_page_addr(), + ax_memory_addr::PAGE_SIZE_4K, + axvm_types::MappingFlags::DEVICE + | axvm_types::MappingFlags::READ + | axvm_types::MappingFlags::WRITE, + ) + } +} + +impl HostTimePlatform for X86_64Arch {} diff --git a/virtualization/axvm/src/arch/x86_64/exit.rs b/virtualization/axvm/src/arch/x86_64/exit.rs new file mode 100644 index 0000000000..4f50ef0437 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/exit.rs @@ -0,0 +1,78 @@ +//! x86-only port, nested-fault, and deferred exit handling. + +use ax_errno::AxResult; +use axvm_types::{AccessWidth, GuestPhysAddr, MappingFlags, Port}; + +use super::{ArchOps, AxvmX86Vcpu, X86_64Arch}; +use crate::architecture::{BoundVcpuExit, VcpuRunAction}; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum DeferredRunWork { + ExternalInterrupt { vector: usize }, + PreemptionTimer, + InterruptEnd { vector: Option }, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct IoReadExit { + pub(crate) port: Port, + pub(crate) width: AccessWidth, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct IoWriteExit { + pub(crate) port: Port, + pub(crate) width: AccessWidth, + pub(crate) data: u64, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct NestedPageFaultExit { + pub(crate) addr: GuestPhysAddr, + pub(crate) access_flags: MappingFlags, +} + +pub(crate) fn handle_io_read( + vm: &crate::AxVM, + vcpu: &crate::vm::AxVCpuRef, + exit: IoReadExit, +) -> AxResult> { + let val = vm.get_devices()?.handle_port_read(exit.port, exit.width)?; + vcpu.set_gpr(0, val); + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn handle_io_write( + vm: &crate::AxVM, + exit: IoWriteExit, +) -> AxResult> { + vm.get_devices()? + .handle_port_write(exit.port, exit.width, exit.data as usize)?; + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn finish( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + work: DeferredRunWork, +) -> AxResult { + match work { + DeferredRunWork::ExternalInterrupt { vector } => { + X86_64Arch::after_external_interrupt(vm, vcpu, vector); + } + DeferredRunWork::PreemptionTimer => { + crate::timer::check_events(); + super::irq::inject_due_pit_irq0(vm, vcpu); + super::irq::inject_pending_serial_irq(vm, vcpu); + } + DeferredRunWork::InterruptEnd { vector } => { + if let Some(vector) = vector { + super::irq::inject_pending_ioapic_irq_after_eoi(vm, vcpu, vector); + } + } + } + Ok(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }) +} diff --git a/virtualization/axvm/src/arch/x86_64/host.rs b/virtualization/axvm/src/arch/x86_64/host.rs deleted file mode 100644 index 99ea4425b4..0000000000 --- a/virtualization/axvm/src/arch/x86_64/host.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Default private ArceOS host adapter for AxVM. - -extern crate alloc; - -use alloc::boxed::Box; -use core::{ - sync::atomic::{AtomicUsize, Ordering}, - time::Duration, -}; - -use ax_errno::AxResult; -use ax_memory_addr::PAGE_SIZE_4K; -use ax_std::{ - os::arceos::{api, modules}, - thread, -}; -use axvm_types::{HostPhysAddr, HostVirtAddr}; - -use crate::{ - arch::{ArchOps, CurrentArch}, - host::{HostConsole, HostCpu, HostMemory, HostPlatform, HostTime}, -}; - -/// Private default host adapter used by [`crate::AxvmRuntime`]. -pub(crate) struct ArceOsHost; - -const CPU_ENABLE_WAIT_TIMEOUT: Duration = Duration::from_secs(5); - -static ARCEOS_HOST: ArceOsHost = ArceOsHost; - -pub(crate) fn arceos_host() -> &'static ArceOsHost { - &ARCEOS_HOST -} - -impl HostMemory for ArceOsHost { - fn alloc_frame(&self) -> Option { - modules::ax_alloc::global_allocator() - .alloc_pages(1, PAGE_SIZE_4K, modules::ax_alloc::UsageKind::PageTable) - .map(|vaddr| self.virt_to_phys(vaddr.into())) - .ok() - } - - fn dealloc_frame(&self, paddr: HostPhysAddr) { - modules::ax_alloc::global_allocator().dealloc_pages( - self.phys_to_virt(paddr).as_usize(), - 1, - modules::ax_alloc::UsageKind::PageTable, - ); - } - - fn alloc_contiguous_frames( - &self, - num_frames: usize, - frame_align: usize, - ) -> Option { - modules::ax_alloc::global_allocator() - .alloc_pages( - num_frames, - frame_align.max(PAGE_SIZE_4K), - modules::ax_alloc::UsageKind::Dma, - ) - .map(|vaddr| self.virt_to_phys(vaddr.into())) - .ok() - } - - fn dealloc_contiguous_frames(&self, paddr: HostPhysAddr, num_frames: usize) { - modules::ax_alloc::global_allocator().dealloc_pages( - self.phys_to_virt(paddr).as_usize(), - num_frames, - modules::ax_alloc::UsageKind::Dma, - ); - } - - fn phys_to_virt(&self, paddr: HostPhysAddr) -> HostVirtAddr { - modules::ax_hal::mem::phys_to_virt(paddr) - } - - fn virt_to_phys(&self, vaddr: HostVirtAddr) -> HostPhysAddr { - modules::ax_hal::mem::virt_to_phys(vaddr) - } -} - -impl HostTime for ArceOsHost { - type CancelToken = usize; - - fn nanos_to_ticks(&self, nanos: u64) -> u64 { - #[cfg(target_arch = "x86_64")] - { - modules::ax_hal::time::nanos_to_ticks(nanos) - } - #[cfg(not(target_arch = "x86_64"))] - { - nanos - } - } - - fn monotonic_time(&self) -> Duration { - modules::ax_hal::time::monotonic_time() - } - - fn set_oneshot_timer(&self, deadline_ns: u64) { - #[cfg(not(target_arch = "loongarch64"))] - modules::ax_hal::time::set_oneshot_timer(deadline_ns); - #[cfg(target_arch = "loongarch64")] - let _ = deadline_ns; - } - - fn register_timer( - &self, - deadline_ns: u64, - callback: Box, - ) -> Self::CancelToken { - crate::timer::register_timer(deadline_ns, callback) - } - - fn cancel_timer(&self, token: Self::CancelToken) { - crate::timer::cancel_timer(token); - } -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn monotonic_time_nanos() -> u64 { - modules::ax_hal::time::monotonic_time_nanos() -} - -#[cfg(target_arch = "aarch64")] -pub(crate) fn handle_host_irq(vector: usize) -> Option { - modules::ax_hal::irq::handle_irq(vector).then_some(vector) -} - -pub(crate) fn dispatch_host_irq(vector: usize) { - modules::ax_hal::irq::handle_irq(vector); -} - -#[cfg(target_arch = "loongarch64")] -pub(crate) fn set_irq_enabled(raw_irq: usize, enabled: bool) { - let gsi = match u32::try_from(raw_irq) { - Ok(gsi) => gsi, - Err(_) => { - warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: out of GSI range"); - return; - } - }; - let irq = match modules::ax_hal::irq::resolve_irq_source( - modules::ax_hal::irq::IrqSource::AcpiGsi(gsi), - ) { - Ok(irq) => irq, - Err(err) => { - warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: {err:?}"); - return; - } - }; - if let Err(err) = modules::ax_hal::irq::set_enable(irq, enabled) { - warn!( - "failed to set LoongArch passthrough IRQ {raw_irq} ({irq:?}) enabled={enabled}: \ - {err:?}" - ); - } -} - -impl HostCpu for ArceOsHost { - type CpuMask = api::task::AxCpuMask; - - fn cpu_count(&self) -> usize { - modules::ax_hal::cpu_num() - } - - fn this_cpu_id(&self) -> usize { - modules::ax_hal::percpu::this_cpu_id() - } -} - -pub(crate) fn cpu_mask_from_raw_bits(bits: usize) -> api::task::AxCpuMask { - api::task::AxCpuMask::from_raw_bits(bits) -} - -pub(crate) type ArceOsCpuMask = api::task::AxCpuMask; -pub(crate) type ArceOsAxTaskExt = modules::ax_task::AxTaskExt; -pub(crate) type ArceOsAxTaskRef = modules::ax_task::AxTaskRef; -pub(crate) type ArceOsCurrentTask = modules::ax_task::CurrentTask; -pub(crate) type ArceOsTaskInner = modules::ax_task::TaskInner; -pub(crate) type ArceOsWaitQueue = modules::ax_task::WaitQueue; -pub(crate) type ArceOsWaitQueueHandle = api::task::AxWaitQueueHandle; -pub(crate) use modules::ax_task::TaskExt as ArceOsTaskExt; - -pub(crate) fn current_task() -> ArceOsCurrentTask { - modules::ax_task::current() -} - -pub(crate) fn spawn_task(task: ArceOsTaskInner) -> ArceOsAxTaskRef { - modules::ax_task::spawn_task(task) -} - -pub(crate) fn yield_now() { - thread::yield_now(); -} - -pub(crate) fn wait_queue_wait_until( - queue: &api::task::AxWaitQueueHandle, - condition: impl Fn() -> bool, -) { - api::task::ax_wait_queue_wait_until(queue, condition, None); -} - -pub(crate) fn wait_queue_wake(queue: &api::task::AxWaitQueueHandle, count: u32) { - api::task::ax_wait_queue_wake(queue, count); -} - -pub(crate) fn send_ipi(cpu_id: usize) { - if modules::ax_hal::percpu::this_cpu_id() == cpu_id { - return; - } - modules::ax_hal::irq::send_ipi( - modules::ax_hal::irq::ipi_irq(), - modules::ax_hal::irq::IpiTarget::Other { cpu_id }, - ); -} - -fn send_ipi_to_all_except_current(cpu_num: usize) { - if cpu_num <= 1 { - return; - } - let cpu_id = modules::ax_hal::percpu::this_cpu_id(); - modules::ax_hal::irq::send_ipi( - modules::ax_hal::irq::ipi_irq(), - modules::ax_hal::irq::IpiTarget::AllExceptCurrent { cpu_id, cpu_num }, - ); -} - -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqContext = modules::ax_hal::irq::IrqContext; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqError = modules::ax_hal::irq::IrqError; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqHandle = modules::ax_hal::irq::IrqHandle; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqId = modules::ax_hal::irq::IrqId; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqReturn = modules::ax_hal::irq::IrqReturn; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqSource = modules::ax_hal::irq::IrqSource; -#[cfg(target_arch = "x86_64")] -pub(crate) fn request_shared_irq( - irq: ArceOsIrqId, - handler: impl FnMut(ArceOsIrqContext) -> ArceOsIrqReturn + Send + 'static, -) -> Result { - modules::ax_hal::irq::request_shared_irq(irq, handler) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn make_irq_id(domain: u16, hwirq: u32) -> ArceOsIrqId { - modules::ax_hal::irq::IrqId::new( - modules::ax_hal::irq::IrqDomainId(domain), - modules::ax_hal::irq::HwIrq(hwirq), - ) -} - -#[cfg(all(target_arch = "x86_64", not(test)))] -pub(crate) fn set_irq_enable(irq: ArceOsIrqId, enabled: bool) -> Result<(), ArceOsIrqError> { - modules::ax_hal::irq::set_enable(irq, enabled) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn resolve_irq_source(source: ArceOsIrqSource) -> Result { - modules::ax_hal::irq::resolve_irq_source(source) -} - -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub fn host_fdt_bootarg() -> usize { - modules::ax_hal::dtb::get_bootarg() -} - -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub fn phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { - modules::ax_hal::mem::phys_to_virt(paddr) -} - -#[cfg(all( - any(feature = "fs", feature = "host-fs"), - any(target_arch = "x86_64", target_arch = "loongarch64") -))] -pub fn shutdown_host_filesystems() -> AxResult { - modules::ax_fs_ng::shutdown_filesystems()?; - let released = modules::ax_fs_ng::release_block_irqs_for_passthrough(); - if released != 0 { - info!("Released {released} host filesystem block IRQ registration(s) before passthrough"); - } - Ok(()) -} - -impl HostConsole for ArceOsHost { - fn write_bytes(&self, bytes: &[u8]) { - modules::ax_hal::console::write_bytes(bytes); - } - - fn read_bytes(&self, bytes: &mut [u8]) -> usize { - modules::ax_hal::console::read_bytes(bytes) - } -} - -impl HostPlatform for ArceOsHost { - fn has_hardware_support(&self) -> bool { - CurrentArch::has_hardware_support() - } - - fn enable_virtualization_on_current_cpu(&self) -> AxResult { - crate::timer::init_percpu(); - crate::percpu::init_current_cpu()?; - crate::percpu::enable_current_cpu()?; - crate::percpu::mark_cpu_enabled(self.this_cpu_id()); - Ok(()) - } - - fn enable_virtualization_on_all_cpus(&self) -> AxResult { - static CORES: AtomicUsize = AtomicUsize::new(0); - - info!("Enabling hardware virtualization support on all cores..."); - CORES.store(0, Ordering::Release); - crate::percpu::reset_enabled_cpu_mask(); - - let cpu_count = self.cpu_count(); - let current_cpu = self.this_cpu_id(); - info!("Core {current_cpu} is initializing hardware virtualization support..."); - self.enable_virtualization_on_current_cpu()?; - info!("Hardware virtualization support enabled on core {current_cpu}"); - CORES.store(1, Ordering::Release); - - for cpu_id in 0..cpu_count { - if cpu_id == current_cpu { - continue; - } - let task = modules::ax_task::TaskInner::new( - move || { - let host = arceos_host(); - info!("Core {cpu_id} is initializing hardware virtualization support..."); - host.enable_virtualization_on_current_cpu() - .expect("failed to enable hardware virtualization"); - info!("Hardware virtualization support enabled on core {cpu_id}"); - let _ = CORES.fetch_add(1, Ordering::Release); - }, - alloc::format!("axvm-hv-init-{cpu_id}"), - modules::ax_task::default_task_stack_size(), - ); - task.set_cpumask(::CpuMask::one_shot(cpu_id)); - modules::ax_task::spawn_task(task); - if cpu_id != self.this_cpu_id() { - send_ipi(cpu_id); - } - } - - info!("Waiting for all cores to enable hardware virtualization..."); - let start = self.monotonic_time(); - let mut wait_rounds = 0usize; - while CORES.load(Ordering::Acquire) != cpu_count { - thread::yield_now(); - wait_rounds = wait_rounds.wrapping_add(1); - if wait_rounds.is_multiple_of(256) { - send_ipi_to_all_except_current(cpu_count); - } - if self.monotonic_time().saturating_sub(start) >= CPU_ENABLE_WAIT_TIMEOUT { - break; - } - } - CurrentArch::register_platform_irq_injector(); - let enabled_count = CORES.load(Ordering::Acquire); - if enabled_count == cpu_count { - info!("All cores have enabled hardware virtualization support."); - } else { - warn!( - "Only {enabled_count}/{cpu_count} cores enabled hardware virtualization before \ - timeout; continuing with host CPU mask {:#x}", - crate::percpu::enabled_cpu_mask() - ); - } - Ok(()) - } -} diff --git a/virtualization/axvm/src/host/irq.rs b/virtualization/axvm/src/arch/x86_64/host_irq.rs similarity index 71% rename from virtualization/axvm/src/host/irq.rs rename to virtualization/axvm/src/arch/x86_64/host_irq.rs index 15d65dd6c5..0da6411cd3 100644 --- a/virtualization/axvm/src/host/irq.rs +++ b/virtualization/axvm/src/arch/x86_64/host_irq.rs @@ -3,23 +3,23 @@ #[cfg(test)] use core::sync::atomic::{AtomicUsize, Ordering}; -use super::arceos; +use ax_std::os::arceos::modules::ax_hal::irq as host_irq; -pub(crate) type IrqContext = arceos::ArceOsIrqContext; -pub(crate) type IrqError = arceos::ArceOsIrqError; -pub(crate) type IrqId = arceos::ArceOsIrqId; -pub(crate) type IrqReturn = arceos::ArceOsIrqReturn; -pub(crate) type IrqSource = arceos::ArceOsIrqSource; +pub(crate) type IrqContext = host_irq::IrqContext; +pub(crate) type IrqError = host_irq::IrqError; +pub(crate) type IrqId = host_irq::IrqId; +pub(crate) type IrqReturn = host_irq::IrqReturn; +pub(crate) type IrqSource = host_irq::IrqSource; pub(crate) fn make_irq_id(domain: u16, hwirq: u32) -> IrqId { - arceos::make_irq_id(domain, hwirq) + host_irq::IrqId::new(host_irq::IrqDomainId(domain), host_irq::HwIrq(hwirq)) } pub(crate) fn request_shared_irq( irq: IrqId, handler: impl FnMut(IrqContext) -> IrqReturn + Send + 'static, -) -> Result { - arceos::request_shared_irq(irq, handler) +) -> Result { + host_irq::request_shared_irq(irq, handler) } #[cfg(test)] @@ -41,11 +41,11 @@ fn set_host_irq_enable_impl(irq: IrqId, enabled: bool) -> Result<(), IrqError> { #[cfg(not(test))] fn set_host_irq_enable_impl(irq: IrqId, enabled: bool) -> Result<(), IrqError> { - arceos::set_irq_enable(irq, enabled) + host_irq::set_enable(irq, enabled) } -pub(crate) fn resolve_irq_source(source: IrqSource) -> Result { - arceos::resolve_irq_source(source) +pub(crate) fn resolve_irq_source(source: IrqSource) -> Result { + host_irq::resolve_irq_source(source) } #[cfg(test)] diff --git a/virtualization/axvm/src/arch/x86_64/irq.rs b/virtualization/axvm/src/arch/x86_64/irq.rs index ce802b629c..970b5862bc 100644 --- a/virtualization/axvm/src/arch/x86_64/irq.rs +++ b/virtualization/axvm/src/arch/x86_64/irq.rs @@ -5,8 +5,8 @@ use axvm_types::VmArchVcpuOps; use crate::{ InterruptTriggerMode, + arch::x86_64::host_irq::{self as irq, IrqSource}, config::VMInterruptMode, - host::irq::{self, IrqSource}, runtime::{VCpuRef, VMRef}, }; @@ -94,7 +94,7 @@ pub fn inject_due_pit_irq0(vm: &VMRef, vcpu: &VCpuRef) { return; } - let now_ns = crate::host::arceos::monotonic_time_nanos(); + let now_ns = ax_std::os::arceos::modules::ax_hal::time::monotonic_time_nanos(); let Ok(devices) = vm.get_devices() else { return; }; @@ -626,7 +626,7 @@ mod tests { IOAPIC_IRQ_PENDING_LEVEL.store(0, Ordering::Release); IOAPIC_IRQ_MASKED.store(0, Ordering::Release); IOAPIC_IRQ_ACTIVATED.store(0, Ordering::Release); - crate::host::irq::reset_test_irq_enable_state(); + crate::arch::x86_64::host_irq::reset_test_irq_enable_state(); for activator in IOAPIC_IRQ_ACTIVATORS.iter() { *activator.lock() = None; } @@ -669,7 +669,7 @@ mod tests { #[test] fn host_irq_storage_preserves_domain_and_hwirq() { - let irq = crate::host::irq::make_irq_id(2, 18); + let irq = crate::arch::x86_64::host_irq::make_irq_id(2, 18); assert_eq!(raw_to_host_irq(host_irq_to_raw(irq)), irq); } @@ -678,7 +678,7 @@ mod tests { with_clean_forwarding_routes(|| { let fallback_guest_gsi = 7; let explicit_guest_gsi = 18; - let host_irq = crate::host::irq::make_irq_id(2, 7); + let host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, 7); IOAPIC_HOST_IRQS[fallback_guest_gsi] .store(host_irq_to_raw(host_irq), Ordering::Release); @@ -693,7 +693,7 @@ mod tests { with_clean_forwarding_routes(|| { let fallback_guest_gsi = 10; let explicit_guest_gsi = 18; - let host_irq = crate::host::irq::make_irq_id(2, 10); + let host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, 10); IOAPIC_HOST_IRQS[fallback_guest_gsi] .store(host_irq_to_raw(host_irq), Ordering::Release); @@ -715,8 +715,8 @@ mod tests { with_clean_forwarding_routes(|| { let low_level_gsi = COM1_GSI; let high_edge_gsi = 18; - let low_host_irq = crate::host::irq::make_irq_id(2, low_level_gsi as u32); - let high_host_irq = crate::host::irq::make_irq_id(2, high_edge_gsi as u32); + let low_host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, low_level_gsi as u32); + let high_host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, high_edge_gsi as u32); register_ioapic_irq_forwarding_route_with_trigger( low_level_gsi, @@ -760,7 +760,7 @@ mod tests { fn forwarding_activator_drops_pre_activation_pending_state() { with_clean_forwarding_routes(|| { let guest_gsi = 18; - let host_irq = crate::host::irq::make_irq_id(2, 10); + let host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, 10); ACTIVATION_COUNT.store(0, Ordering::Release); register_ioapic_irq_forwarding_route(guest_gsi, host_irq); register_ioapic_irq_forwarding_activator(guest_gsi, count_activation); @@ -773,7 +773,7 @@ mod tests { forwarded_ioapic_gsi_state_for_test(guest_gsi), (false, false, false) ); - assert!(crate::host::irq::test_irq_is_enabled(host_irq)); + assert!(crate::arch::x86_64::host_irq::test_irq_is_enabled(host_irq)); }); } diff --git a/virtualization/axvm/src/arch/x86_64/mod.rs b/virtualization/axvm/src/arch/x86_64/mod.rs index 6f6c30a8e3..d35ae743f4 100644 --- a/virtualization/axvm/src/arch/x86_64/mod.rs +++ b/virtualization/axvm/src/arch/x86_64/mod.rs @@ -23,20 +23,28 @@ use x86_vlapic::{ }; use super::{ - ArchOps, BoundVcpuExit, HypercallExit, IoReadExit, IoWriteExit, LegacyDeferredRunWork, - MmioReadExit, MmioWriteExit, NestedPageFaultExit, SysRegReadExit, SysRegWriteExit, - VcpuCreateContext, VcpuRunAction, VcpuSetupContext, + ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, + VcpuRunAction, VcpuSetupContext, }; use crate::{ StopReason, - host::{HostConsole, HostMemory, HostTime, default_host}, + host::{HostMemory, HostTime, default_host}, manager, vcpu::get_current_vcpu, }; +pub(crate) mod boot; +mod capabilities; +mod exit; +mod host_irq; pub(crate) mod irq; mod npt; pub(crate) mod port; +#[path = "../../architecture/sysreg.rs"] +mod sysreg; + +use exit::{DeferredRunWork, IoReadExit, IoWriteExit, NestedPageFaultExit}; +use sysreg::{SysRegReadExit, SysRegWriteExit}; const QEMU_EXIT_PORT: u16 = 0x604; const QEMU_EXIT_MAGIC: u64 = 0x2000; @@ -50,7 +58,7 @@ impl ArchOps for X86_64Arch { type VCpu = AxvmX86Vcpu; type PerCpu = AxvmX86PerCpu; type VcpuCreateState = X86VcpuCreateState; - type DeferredRunWork = LegacyDeferredRunWork; + type DeferredRunWork = DeferredRunWork; type NestedPageTable = npt::NestedPageTable; fn has_hardware_support() -> bool { @@ -65,18 +73,25 @@ impl ArchOps for X86_64Arch { fn build_vcpu_create_config( _state: &Self::VcpuCreateState, - _ctx: VcpuCreateContext, + ctx: VcpuCreateContext, ) -> AxResult<::CreateConfig> { + let _ = ctx.into_parts(); Ok(X86VCpuCreateConfig) } fn build_vcpu_setup_config( ctx: VcpuSetupContext<'_>, ) -> AxResult<::SetupConfig> { + let ( + _interrupt_mode, + emulates_console, + passthrough_ports, + memory_regions, + _firmware_boot, + ) = ctx.into_parts(); let mut config = X86VCpuSetupConfig { - emulate_com1: ctx.emulates_console, - guest_memory_regions: ctx - .memory_regions + emulate_com1: emulates_console, + guest_memory_regions: memory_regions .iter() .map(|region| x86_vcpu::X86GuestMemoryRegion { gpa: X86GuestPhysAddr::from_usize(region.gpa.as_usize()), @@ -86,7 +101,7 @@ impl ArchOps for X86_64Arch { .collect(), ..Default::default() }; - for port in ctx.passthrough_ports { + for port in passthrough_ports { x86_result(config.add_passthrough_port_range(port.base, port.length))?; } Ok(config) @@ -115,26 +130,6 @@ impl ArchOps for X86_64Arch { irq::inject_pending_serial_irq(vm, vcpu); } - fn after_preemption_timer(vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { - crate::timer::check_events(); - irq::inject_due_pit_irq0(vm, vcpu); - irq::inject_pending_serial_irq(vm, vcpu); - } - - fn after_interrupt_end( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - vector: Option, - ) { - if let Some(vector) = vector { - irq::inject_pending_ioapic_irq_after_eoi(vm, vcpu, vector); - } - } - - fn handle_halt() -> VcpuRunAction { - VcpuRunAction::Yield - } - fn on_last_vcpu_exit(vm_id: usize) { irq::disable_ioapic_irq_forwarding_for_vm(vm_id); } @@ -148,7 +143,7 @@ impl ArchOps for X86_64Arch { X86VmExit::Hypercall { nr, args } => { super::handle_hypercall(vm, vcpu, HypercallExit { nr, args }) } - X86VmExit::PortIoRead { port, width } => super::handle_io_read::( + X86VmExit::PortIoRead { port, width } => exit::handle_io_read( vm, vcpu, IoReadExit { @@ -159,11 +154,12 @@ impl ArchOps for X86_64Arch { X86VmExit::PortIoWrite { port, width, data } => { if x86_qemu_shutdown_port(port, width, data) { warn!("VM[{}] run VCpu[{}] SystemDown", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Stop( - StopReason::SystemDown, - ))) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: Some(StopReason::SystemDown), + })) } else { - super::handle_io_write( + exit::handle_io_write( vm, IoWriteExit { port: x86_port_to_ax(port), @@ -198,7 +194,7 @@ impl ArchOps for X86_64Arch { data, }, ), - X86VmExit::MsrRead { addr } => super::handle_sys_reg_read( + X86VmExit::MsrRead { addr } => sysreg::handle_read( vm, vcpu, SysRegReadExit { @@ -206,47 +202,47 @@ impl ArchOps for X86_64Arch { reg: 0, }, ), - X86VmExit::MsrWrite { addr, value } => super::handle_sys_reg_write( + X86VmExit::MsrWrite { addr, value } => sysreg::handle_write( vm, SysRegWriteExit { addr: x86_msr_addr_to_ax(addr), value, }, ), - X86VmExit::NestedPageFault { addr, access_flags } => { - handle_x86_nested_page_fault::( - vm, - NestedPageFaultExit { - addr: x86_guest_phys_addr_to_ax(addr), - access_flags: x86_access_flags_to_ax(access_flags), - }, - ) - } + X86VmExit::NestedPageFault { addr, access_flags } => handle_x86_nested_page_fault( + vm, + NestedPageFaultExit { + addr: x86_guest_phys_addr_to_ax(addr), + access_flags: x86_access_flags_to_ax(access_flags), + }, + ), X86VmExit::ExternalInterrupt { vector } => { debug!("VM[{}] run VCpu[{}] get irq {vector}", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Defer( - LegacyDeferredRunWork::ExternalInterrupt { - vector: vector as usize, - }, - )) + Ok(BoundVcpuExit::Defer(DeferredRunWork::ExternalInterrupt { + vector: vector as usize, + })) } X86VmExit::PreemptionTimer => { - Ok(BoundVcpuExit::Defer(LegacyDeferredRunWork::PreemptionTimer)) + Ok(BoundVcpuExit::Defer(DeferredRunWork::PreemptionTimer)) } X86VmExit::InterruptEnd { vector } => { - Ok(BoundVcpuExit::Defer(LegacyDeferredRunWork::InterruptEnd { + Ok(BoundVcpuExit::Defer(DeferredRunWork::InterruptEnd { vector, })) } X86VmExit::Halt => { debug!("VM[{}] run VCpu[{}] Halt", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(Self::handle_halt())) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } X86VmExit::SystemDown => { warn!("VM[{}] run VCpu[{}] SystemDown", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Stop( - StopReason::SystemDown, - ))) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: Some(StopReason::SystemDown), + })) } X86VmExit::FailEntry { hardware_entry_failure_reason, @@ -256,7 +252,10 @@ impl ArchOps for X86_64Arch { vm.id(), vcpu.id() ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } X86VmExit::Nothing => Ok(BoundVcpuExit::Continue), _ => Err(AxError::Unsupported), @@ -268,7 +267,7 @@ impl ArchOps for X86_64Arch { vcpu: &crate::vm::AxVCpuRef, work: Self::DeferredRunWork, ) -> AxResult { - super::finish_legacy_deferred_run_work::(vm, vcpu, work) + exit::finish(vm, vcpu, work) } } @@ -296,7 +295,7 @@ impl X86VlapicHostOps for AxvmX86HostOps { } fn current_time_nanos() -> u64 { - crate::host::arceos::monotonic_time_nanos() + ax_std::os::arceos::modules::ax_hal::time::monotonic_time_nanos() } fn register_timer(deadline_nanos: u64, callback: X86TimerCallback) -> Option { @@ -307,15 +306,15 @@ impl X86VlapicHostOps for AxvmX86HostOps { } fn cancel_timer(token: usize) { - default_host().cancel_timer(token); + crate::timer::cancel_timer(token); } fn write_bytes(bytes: &[u8]) { - default_host().write_bytes(bytes); + ax_std::os::arceos::modules::ax_hal::console::write_bytes(bytes); } fn read_bytes(bytes: &mut [u8]) -> usize { - default_host().read_bytes(bytes) + ax_std::os::arceos::modules::ax_hal::console::read_bytes(bytes) } fn current_vm_id() -> X86VmId { @@ -387,7 +386,7 @@ impl X86HostOps for AxvmX86HostOps { } fn nanos_to_ticks(nanos: u64) -> u64 { - default_host().nanos_to_ticks(nanos) + ax_std::os::arceos::modules::ax_hal::time::nanos_to_ticks(nanos) } fn poll_host_interrupt() -> Option { @@ -528,20 +527,20 @@ pub(crate) fn x86_apic_access_page_addr() -> axvm_types::HostPhysAddr { axvm_types::HostPhysAddr::from(addr.as_usize()) } -fn handle_x86_nested_page_fault( +fn handle_x86_nested_page_fault( vm: &crate::AxVMRef, exit: NestedPageFaultExit, -) -> AxResult> -where - A: ArchOps, -{ +) -> AxResult> { if vm.get_devices()?.find_mmio_dev(exit.addr).is_some() { warn!( "VM[{}] nested page fault at {:#x} maps MMIO but x86 core did not decode it", vm.id(), exit.addr.as_usize() ); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); + return Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })); } if vm.handle_nested_page_fault(exit.addr, exit.access_flags) { @@ -553,7 +552,10 @@ where exit.addr.as_usize(), exit.access_flags ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } } diff --git a/virtualization/axvm/src/arch/x86_64/npt.rs b/virtualization/axvm/src/arch/x86_64/npt.rs index 906bbecfbf..e097289f86 100644 --- a/virtualization/axvm/src/arch/x86_64/npt.rs +++ b/virtualization/axvm/src/arch/x86_64/npt.rs @@ -327,12 +327,8 @@ pub type ExtendedPageTableEntry = EPTEntry; #[cfg(feature = "svm")] pub type ExtendedPageTableEntry = NPTEntry; -pub(crate) type NestedPageTable = crate::arch::npt::LeveledPageTable< - ExtendedPageTableMetadata, - ExtendedPageTableMetadata, - H, - false, ->; +pub(crate) type NestedPageTable = + crate::npt::LeveledPageTable; fn config_to_flags(config: ptg::PteConfig) -> MappingFlags { let mut flags = MappingFlags::empty(); diff --git a/virtualization/axvm/src/architecture/capabilities.rs b/virtualization/axvm/src/architecture/capabilities.rs new file mode 100644 index 0000000000..52d407971a --- /dev/null +++ b/virtualization/axvm/src/architecture/capabilities.rs @@ -0,0 +1,75 @@ +//! Small capability boundaries implemented by the selected guest architecture. + +use ax_errno::AxResult; + +use super::ArchOps; + +/// Architecture-specific guest image planning layered over common byte loading. +pub(crate) trait BootImagePlatform { + fn load_images_from_memory( + loader: &mut crate::boot::images::ImageLoaderCore<'_>, + images: crate::boot::StaticVmImage, + ) -> AxResult { + loader.load_standard_images_from_memory(images, Self::load_guest_dtb) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + fn load_images_from_filesystem( + loader: &mut crate::boot::images::ImageLoaderCore<'_>, + ) -> AxResult { + loader.load_standard_images_from_filesystem(Self::load_guest_dtb) + } + + fn load_guest_dtb( + _loader: &crate::boot::images::ImageLoaderCore<'_>, + _dtb: &crate::boot::fdt::GuestDtbImage, + ) -> AxResult { + Ok(()) + } + + fn is_x86_linux_image_config( + _config: &axvmconfig::AxVMCrateConfig, + _provider: &dyn crate::boot::BootImageProvider, + ) -> bool { + false + } +} + +/// Architecture-owned interrupt fabric and device registration hooks. +pub(crate) trait DevicePlatform { + fn configure_interrupt_fabric( + _factories: &mut axdevice::DeviceFactoryRegistry, + mode: axvm_types::VMInterruptMode, + _configs: &[axvm_types::EmulatedDeviceConfig], + ) -> AxResult { + Ok(crate::InterruptFabric::new(mode)) + } + + fn register_devices( + _vm: &crate::AxVM, + _config: &crate::config::AxVMConfig, + _devices: &mut axdevice::AxVmDevices, + ) -> AxResult { + Ok(()) + } +} + +/// Architecture-owned guest address-space additions. +pub(crate) trait AddressSpacePlatform: ArchOps { + fn append_owned_regions(_regions: &mut alloc::vec::Vec) {} + + fn map_address_space( + _address_space: &mut axaddrspace::AddrSpace, + ) -> AxResult { + Ok(()) + } +} + +/// Architecture-specific host timer policy used by the ArceOS adapter. +pub(crate) trait HostTimePlatform { + fn set_oneshot_timer(deadline_ns: u64) { + ax_std::os::arceos::modules::ax_hal::time::set_oneshot_timer(deadline_ns); + } + + fn register_timer_callback() {} +} diff --git a/virtualization/axvm/src/architecture/cpu_up.rs b/virtualization/axvm/src/architecture/cpu_up.rs new file mode 100644 index 0000000000..7d9e69f218 --- /dev/null +++ b/virtualization/axvm/src/architecture/cpu_up.rs @@ -0,0 +1,67 @@ +//! Shared secondary-vCPU boot flow for architectures that expose CPU-up exits. + +use ax_errno::AxResult; +use axvm_types::GuestPhysAddr; + +use crate::architecture::{ArchOps, BoundVcpuExit, VcpuRunAction}; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct CpuUpExit { + pub(crate) target_cpu: u64, + pub(crate) entry_point: GuestPhysAddr, + pub(crate) arg: u64, +} + +pub(crate) trait CpuUpOps: ArchOps { + fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { + vcpu.set_gpr(0, 0); + } + + fn target_vcpu_id(vm: &crate::AxVMRef, target_cpu: u64) -> Option { + vm.get_vcpu_affinities_pcpu_ids() + .iter() + .find_map(|(vcpu_id, _, phys_id)| (*phys_id == target_cpu as usize).then_some(*vcpu_id)) + } +} + +pub(crate) fn handle( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + exit: CpuUpExit, +) -> AxResult> { + let vm_id = vm.id(); + let vcpu_id = vcpu.id(); + info!( + "VM[{vm_id}]'s VCpu[{vcpu_id}] try to boot target_cpu [{}] entry_point={:x} arg={:#x}", + exit.target_cpu, exit.entry_point, exit.arg + ); + + let Some(target_vcpu_id) = A::target_vcpu_id(vm, exit.target_cpu) else { + warn!( + "VM[{vm_id}] cannot resolve architecture CPU target {} to a VM-local vCPU", + exit.target_cpu + ); + vcpu.set_return_value(usize::MAX); + return Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })); + }; + + match crate::runtime::vcpus::vcpu_on( + vm.clone(), + target_vcpu_id, + exit.entry_point, + exit.arg as _, + ) { + Ok(()) => A::set_cpu_up_success(vcpu), + Err(err) => { + warn!("Failed to boot VM[{vm_id}] VCpu[{target_vcpu_id}]: {err:?}"); + vcpu.set_return_value(usize::MAX); + } + } + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) +} diff --git a/virtualization/axvm/src/architecture/exit.rs b/virtualization/axvm/src/architecture/exit.rs new file mode 100644 index 0000000000..2c1d95c6f6 --- /dev/null +++ b/virtualization/axvm/src/architecture/exit.rs @@ -0,0 +1,58 @@ +//! Architecture-neutral handlers for exits shared by every guest architecture. + +use ax_errno::AxResult; +use axvm_types::VmArchVcpuOps; + +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; + +pub(crate) fn handle_mmio_read( + vm: &crate::AxVM, + vcpu: &crate::vm::AxVCpuRef, + exit: MmioReadExit, +) -> AxResult> { + let raw = vm.get_devices()?.handle_mmio_read(exit.addr, exit.width)?; + let masked = raw & crate::vm::width_mask(exit.width); + let val = if exit.signed_ext { + crate::vm::sign_extend_value(masked, exit.width) + } else { + masked & crate::vm::width_mask(exit.reg_width) + }; + vcpu.set_gpr(exit.reg, val); + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn handle_mmio_write( + vm: &crate::AxVMRef, + exit: MmioWriteExit, +) -> AxResult> { + vm.handle_mmio_write(exit.addr, exit.width, exit.data as usize)?; + A::after_mmio_write(vm); + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn handle_hypercall( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + exit: HypercallExit, +) -> AxResult> { + debug!("Hypercall [{:#x}] args {:x?}", exit.nr, exit.args); + match crate::runtime::hvc::HyperCall::new(vm.clone(), exit.nr, exit.args) { + Ok(hypercall) => { + let ret_val = match hypercall.execute() { + Ok(ret_val) => ret_val as isize, + Err(err) => { + warn!("Hypercall [{:#x}] failed: {err:?}", exit.nr); + -1 + } + }; + vcpu.set_return_value(ret_val as usize); + } + Err(err) => { + warn!("Hypercall [{:#x}] failed: {err:?}", exit.nr); + } + } + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) +} diff --git a/virtualization/axvm/src/architecture/mod.rs b/virtualization/axvm/src/architecture/mod.rs new file mode 100644 index 0000000000..891b3601d4 --- /dev/null +++ b/virtualization/axvm/src/architecture/mod.rs @@ -0,0 +1,16 @@ +//! Architecture-neutral contracts shared by target implementations. + +mod capabilities; +mod exit; +pub(crate) mod ops; +mod types; + +pub(crate) use capabilities::{ + AddressSpacePlatform, BootImagePlatform, DevicePlatform, HostTimePlatform, +}; +pub(crate) use exit::{handle_hypercall, handle_mmio_read, handle_mmio_write}; +pub(crate) use ops::ArchOps; +pub(crate) use types::{ + BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, VcpuRunAction, + VcpuSetupContext, +}; diff --git a/virtualization/axvm/src/architecture/ops.rs b/virtualization/axvm/src/architecture/ops.rs new file mode 100644 index 0000000000..5774240fc4 --- /dev/null +++ b/virtualization/axvm/src/architecture/ops.rs @@ -0,0 +1,244 @@ +//! Core vCPU and nested-paging contract implemented by every target architecture. + +use alloc::{format, vec::Vec}; + +use ax_errno::{AxResult, ax_err}; +use ax_memory_addr::{PhysAddr, VirtAddr}; +use axaddrspace::NestedPageTableOps; +use axvm_types::{NestedPagingConfig, VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState}; + +use super::{BoundVcpuExit, VcpuCreateContext, VcpuRunAction, VcpuSetupContext}; + +pub(crate) trait ArchOps { + type VCpu: VmArchVcpuOps; + type PerCpu: VmArchPerCpuOps; + type VcpuCreateState; + type DeferredRunWork; + type NestedPageTable: NestedPageTableOps; + + fn has_hardware_support() -> bool; + + fn max_guest_page_table_levels() -> usize { + 4 + } + + fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { + let mut levels = Self::max_guest_page_table_levels(); + for cpu_id in target_phys_cpu_ids(vcpu_mappings) { + levels = levels.min( + crate::percpu::cpu_max_guest_page_table_levels(cpu_id) + .unwrap_or_else(Self::max_guest_page_table_levels), + ); + } + Ok(levels) + } + + fn nested_paging_config( + root_paddr: PhysAddr, + levels: usize, + _vcpu_mappings: &[(usize, Option, usize)], + ) -> AxResult { + let gpa_bits = match levels { + 3 => 39, + 4 => 48, + _ => return ax_errno::ax_err!(InvalidInput, "unsupported nested page-table levels"), + }; + Ok(NestedPagingConfig::new(root_paddr, levels, gpa_bits, 0)) + } + + fn new_nested_page_table(levels: usize) -> AxResult; + + fn clean_dcache_range(_addr: VirtAddr, _size: usize) {} + + fn new_vcpu_create_state( + vcpu_mappings: &[(usize, Option, usize)], + ) -> AxResult; + + fn build_vcpu_create_config( + state: &Self::VcpuCreateState, + ctx: VcpuCreateContext, + ) -> AxResult<::CreateConfig>; + + fn build_vcpu_setup_config( + ctx: VcpuSetupContext<'_>, + ) -> AxResult<::SetupConfig>; + + fn register_platform_irq_injector() {} + + fn vcpu_affinities( + cpu_num: usize, + phys_cpu_ids: Option<&[usize]>, + phys_cpu_sets: Option<&[usize]>, + ) -> Vec<(usize, Option, usize)> { + default_vcpu_affinities(cpu_num, phys_cpu_ids, phys_cpu_sets) + } + + fn set_vcpu_on_args(vcpu: &crate::vm::AxVCpuRef, _vcpu_id: usize, arg: usize) { + vcpu.set_gpr(0, arg); + } + + fn before_first_run(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) {} + + fn before_vcpu_run(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) {} + + fn inject_pending_interrupt( + _vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + interrupt: crate::vm::PendingInterrupt, + ) { + match interrupt { + crate::vm::PendingInterrupt::Normal(vector) => { + trace!( + "Injecting queued interrupt {vector:#x} into VM[{}] VCpu[{}]", + vcpu.vm_id(), + vcpu.id() + ); + if let Err(err) = vcpu.inject_interrupt(vector) { + warn!( + "Failed to inject queued interrupt {vector:#x} into VM[{}] VCpu[{}]: \ + {err:?}", + vcpu.vm_id(), + vcpu.id() + ); + } + } + crate::vm::PendingInterrupt::External { + vector, + physical_irq, + } => { + warn!( + "VM[{}] VCpu[{}] dropped unsupported external interrupt vector={vector:#x}, \ + physical_irq={physical_irq:#x}", + vcpu.vm_id(), + vcpu.id() + ); + } + } + } + + fn after_external_interrupt( + _vm: &crate::AxVMRef, + _vcpu: &crate::vm::AxVCpuRef, + vector: usize, + ) { + crate::host::arceos::dispatch_host_irq(vector); + crate::check_timer_events(); + } + + fn on_last_vcpu_exit(_vm_id: usize) {} + + fn after_mmio_write(_vm: &crate::AxVMRef) {} + + fn handle_vcpu_exit_bound( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + exit: ::Exit, + ) -> AxResult>; + + fn finish_deferred_run_work( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + work: Self::DeferredRunWork, + ) -> AxResult; + + fn run_vcpu( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + ) -> AxResult + where + Self: Sized, + { + let vm_id = vm.id(); + let vcpu_id = vcpu.id(); + + match vcpu.state() { + VmVcpuState::Free => vcpu.bind()?, + VmVcpuState::Ready => {} + state => { + return ax_err!( + BadState, + format!("VCpu state is not Free or Ready, but {state:?}") + ); + } + } + + let run_result = vcpu.with_current_cpu_set(|| -> AxResult<_> { + loop { + crate::runtime::vcpus::inject_pending_interrupts::(vm.id(), vcpu_id, vcpu); + + let exit = vcpu.run()?; + trace!("{exit:#x?}"); + match Self::handle_vcpu_exit_bound(vm, vcpu, exit)? { + BoundVcpuExit::Continue => continue, + action => break Ok(action), + } + } + }); + + let unbind_result = vcpu.unbind(); + match run_result { + Ok(BoundVcpuExit::Complete(action)) => { + unbind_result?; + Ok(action) + } + Ok(BoundVcpuExit::Defer(work)) => { + unbind_result?; + Self::finish_deferred_run_work(vm, vcpu, work) + } + Ok(BoundVcpuExit::Continue) => unreachable!("continued exits do not leave run loop"), + Err(err) => { + if let Err(unbind_err) = unbind_result { + warn!( + "VM[{vm_id}] VCpu[{vcpu_id}] unbind after run error failed: {unbind_err:?}" + ); + } + Err(err) + } + } + } +} + +pub(crate) fn target_phys_cpu_ids(vcpu_mappings: &[(usize, Option, usize)]) -> Vec { + let mut cpu_ids = Vec::new(); + for (_, maybe_mask, phys_id) in vcpu_mappings { + if let Some(mask) = maybe_mask { + for cpu_id in 0..usize::BITS as usize { + if mask & (1usize << cpu_id) != 0 && !cpu_ids.contains(&cpu_id) { + cpu_ids.push(cpu_id); + } + } + } else if !cpu_ids.contains(phys_id) { + cpu_ids.push(*phys_id); + } + } + cpu_ids +} + +pub(crate) fn default_vcpu_affinities( + cpu_num: usize, + phys_cpu_ids: Option<&[usize]>, + phys_cpu_sets: Option<&[usize]>, +) -> Vec<(usize, Option, usize)> { + let mut vcpus = Vec::with_capacity(cpu_num); + for vcpu_id in 0..cpu_num { + vcpus.push((vcpu_id, None, vcpu_id)); + } + + if let Some(phys_cpu_sets) = phys_cpu_sets { + for (vcpu_id, pcpu_mask_bitmap) in phys_cpu_sets.iter().enumerate() { + if let Some(vcpu) = vcpus.get_mut(vcpu_id) { + vcpu.1 = Some(*pcpu_mask_bitmap); + } + } + } + + if let Some(phys_cpu_ids) = phys_cpu_ids { + for (vcpu_id, phys_id) in phys_cpu_ids.iter().enumerate() { + if let Some(vcpu) = vcpus.get_mut(vcpu_id) { + vcpu.2 = *phys_id; + } + } + } + + vcpus +} diff --git a/virtualization/axvm/src/architecture/sysreg.rs b/virtualization/axvm/src/architecture/sysreg.rs new file mode 100644 index 0000000000..1ab7ff7ae1 --- /dev/null +++ b/virtualization/axvm/src/architecture/sysreg.rs @@ -0,0 +1,39 @@ +//! Shared system-register device exits used by AArch64 and x86_64 guests. + +use ax_errno::AxResult; +use axvm_types::{AccessWidth, SysRegAddr, VmArchVcpuOps}; + +use crate::architecture::BoundVcpuExit; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct SysRegReadExit { + pub(crate) addr: SysRegAddr, + pub(crate) reg: usize, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct SysRegWriteExit { + pub(crate) addr: SysRegAddr, + pub(crate) value: u64, +} + +pub(crate) fn handle_read( + vm: &crate::AxVM, + vcpu: &crate::vm::AxVCpuRef, + exit: SysRegReadExit, +) -> AxResult> { + let val = vm + .get_devices()? + .handle_sys_reg_read(exit.addr, AccessWidth::Qword)?; + vcpu.set_gpr(exit.reg, val); + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn handle_write( + vm: &crate::AxVM, + exit: SysRegWriteExit, +) -> AxResult> { + vm.get_devices()? + .handle_sys_reg_write(exit.addr, AccessWidth::Qword, exit.value as usize)?; + Ok(BoundVcpuExit::Continue) +} diff --git a/virtualization/axvm/src/architecture/types.rs b/virtualization/axvm/src/architecture/types.rs new file mode 100644 index 0000000000..d48b8d5d48 --- /dev/null +++ b/virtualization/axvm/src/architecture/types.rs @@ -0,0 +1,91 @@ +//! Architecture-neutral vCPU contexts and normalized runtime actions. + +use axvm_types::{AccessWidth, GuestPhysAddr, PassThroughPortConfig, VMInterruptMode}; + +use crate::StopReason; + +/// Scheduler effects selected after an architecture-local vCPU exit. +#[derive(Debug)] +pub(crate) struct VcpuRunAction { + pub(crate) waits_for_event: bool, + pub(crate) stop_reason: Option, +} + +/// Result of handling one exit while the vCPU is still bound to the host CPU. +#[derive(Debug)] +pub(crate) enum BoundVcpuExit { + /// The exit was handled completely; re-enter the guest in the current run slice. + Continue, + /// The run slice is complete and can return this scheduler action after unbind. + Complete(VcpuRunAction), + /// Finish architecture-local work after unbinding the vCPU. + Defer(D), +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct MmioReadExit { + pub(crate) addr: GuestPhysAddr, + pub(crate) width: AccessWidth, + pub(crate) reg: usize, + pub(crate) reg_width: AccessWidth, + pub(crate) signed_ext: bool, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct MmioWriteExit { + pub(crate) addr: GuestPhysAddr, + pub(crate) width: AccessWidth, + pub(crate) data: u64, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct HypercallExit { + pub(crate) nr: u64, + pub(crate) args: [u64; 6], +} + +pub(crate) struct VcpuCreateContext { + pub(crate) vcpu_id: usize, + pub(crate) phys_cpu_id: usize, + pub(crate) dtb_addr: Option, + pub(crate) firmware_boot: bool, +} + +impl VcpuCreateContext { + pub(crate) fn into_parts(self) -> (usize, usize, Option, bool) { + ( + self.vcpu_id, + self.phys_cpu_id, + self.dtb_addr, + self.firmware_boot, + ) + } +} + +pub(crate) struct VcpuSetupContext<'a> { + pub(crate) interrupt_mode: VMInterruptMode, + pub(crate) emulates_console: bool, + pub(crate) passthrough_ports: &'a [PassThroughPortConfig], + pub(crate) memory_regions: &'a [crate::vm::VMMemoryRegion], + pub(crate) firmware_boot: bool, +} + +impl<'a> VcpuSetupContext<'a> { + pub(crate) fn into_parts( + self, + ) -> ( + VMInterruptMode, + bool, + &'a [PassThroughPortConfig], + &'a [crate::vm::VMMemoryRegion], + bool, + ) { + ( + self.interrupt_mode, + self.emulates_console, + self.passthrough_ports, + self.memory_regions, + self.firmware_boot, + ) + } +} diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/device.rs b/virtualization/axvm/src/boot/fdt/device.rs similarity index 100% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/device.rs rename to virtualization/axvm/src/boot/fdt/device.rs diff --git a/virtualization/axvm/src/boot/fdt/mod.rs b/virtualization/axvm/src/boot/fdt/mod.rs new file mode 100644 index 0000000000..184157b52e --- /dev/null +++ b/virtualization/axvm/src/boot/fdt/mod.rs @@ -0,0 +1,6 @@ +//! Shared FDT source root and the selected architecture's guest boot facade. + +#[path = "../../arch/aarch64/boot/fdt/mod.rs"] +mod platform; + +pub use platform::*; diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/print.rs b/virtualization/axvm/src/boot/fdt/print.rs similarity index 100% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/print.rs rename to virtualization/axvm/src/boot/fdt/print.rs diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/tree.rs b/virtualization/axvm/src/boot/fdt/tree.rs similarity index 99% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/tree.rs rename to virtualization/axvm/src/boot/fdt/tree.rs index c5a6ea1640..f47dcf63bd 100644 --- a/virtualization/axvm/src/arch/aarch64/boot/fdt/tree.rs +++ b/virtualization/axvm/src/boot/fdt/tree.rs @@ -39,7 +39,6 @@ impl FdtTree { &self.fdt } - #[cfg(any(test, target_arch = "aarch64"))] pub(crate) fn inner_mut(&mut self) -> &mut Fdt { &mut self.fdt } diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/tree_tests.rs b/virtualization/axvm/src/boot/fdt/tree_tests.rs similarity index 100% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/tree_tests.rs rename to virtualization/axvm/src/boot/fdt/tree_tests.rs diff --git a/virtualization/axvm/src/boot/guest_platform/mod.rs b/virtualization/axvm/src/boot/guest_platform/mod.rs new file mode 100644 index 0000000000..c8346477b4 --- /dev/null +++ b/virtualization/axvm/src/boot/guest_platform/mod.rs @@ -0,0 +1,5 @@ +//! Compatibility namespace for architecture-owned guest platform resources. + +pub mod loongarch64 { + pub use crate::arch::guest_platform::*; +} diff --git a/virtualization/axvm/src/arch/x86_64/boot/images/linux.rs b/virtualization/axvm/src/boot/images/linux.rs similarity index 100% rename from virtualization/axvm/src/arch/x86_64/boot/images/linux.rs rename to virtualization/axvm/src/boot/images/linux.rs diff --git a/virtualization/axvm/src/boot/images/mod.rs b/virtualization/axvm/src/boot/images/mod.rs new file mode 100644 index 0000000000..132250fb25 --- /dev/null +++ b/virtualization/axvm/src/boot/images/mod.rs @@ -0,0 +1,329 @@ +//! Architecture-neutral guest image loading and source access. + +use alloc::format; + +use ax_errno::{AxResult, ax_err, ax_err_type}; +use axvmconfig::AxVMCrateConfig; +use byte_unit::Byte; + +use super::{BootImageProvider, StaticVmImage}; +use crate::{AxVMRef, GuestPhysAddr, VMMemoryRegion}; + +mod linux; + +pub use crate::arch::ImageLoader; + +/// Return whether an x86 configuration selects direct Linux bzImage boot. +/// +/// This returns `false` on non-x86 targets. +pub fn is_x86_linux_image_config( + config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> bool { + crate::arch::is_x86_linux_image_config(config, provider) +} + +/// Return the q35 PCI INTx route reserved for the passthrough block device. +pub const fn x86_qemu_passthrough_block_intx() -> (u8, u8, u8, usize) { + (3, 0, 1, 19) +} + +pub fn get_image_header( + config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> Option { + match config.kernel.image_location.as_deref() { + Some("memory") => with_memory_image(config, provider, linux::Header::parse).flatten(), + #[cfg(any(feature = "fs", feature = "host-fs"))] + Some("fs") => { + let data = fs::kernel_read(config, provider, linux::Header::hdr_size()).ok()?; + linux::Header::parse(&data) + } + _ => None, + } +} + +pub(crate) struct ImageLoaderCore<'a> { + pub(crate) provider: &'a dyn BootImageProvider, + pub(crate) main_memory: VMMemoryRegion, + pub(crate) vm: AxVMRef, + pub(crate) config: AxVMCrateConfig, + guest_dtb: Option, + pub(crate) kernel_load_gpa: GuestPhysAddr, + pub(crate) bios_load_gpa: Option, + pub(crate) ramdisk_load_gpa: Option, +} + +impl<'a> ImageLoaderCore<'a> { + pub(crate) fn new( + main_memory: VMMemoryRegion, + config: AxVMCrateConfig, + vm: AxVMRef, + provider: &'a dyn BootImageProvider, + guest_dtb: Option, + ) -> Self { + Self { + provider, + main_memory, + vm, + config, + guest_dtb, + kernel_load_gpa: GuestPhysAddr::default(), + bios_load_gpa: None, + ramdisk_load_gpa: None, + } + } + + pub(crate) fn load(&mut self) -> AxResult { + self.config.kernel.validate_boot_config()?; + debug!( + "Loading VM[{}] images into memory region: gpa={:#x}, hva={:#x}, size={:#}", + self.vm.id(), + self.main_memory.gpa, + self.main_memory.hva, + Byte::from(self.main_memory.size()) + ); + self.capture_prepared_load_addresses(); + + match self.config.kernel.image_location.as_deref() { + Some("memory") => { + let images = memory_images_for_vm(&self.config, self.provider)?; + crate::arch::load_images_from_memory(self, images) + } + #[cfg(any(feature = "fs", feature = "host-fs"))] + Some("fs") => crate::arch::load_images_from_filesystem(self), + _ => ax_err!( + InvalidInput, + "Unsupported image_location; use \"memory\" or enable fs feature for \"fs\"" + ), + } + } + + pub(crate) fn load_standard_images_from_memory( + &mut self, + images: StaticVmImage, + load_guest_dtb: fn(&Self, &crate::boot::fdt::GuestDtbImage) -> AxResult, + ) -> AxResult { + load_vm_image_from_memory(images.kernel, self.kernel_load_gpa, self.vm.clone())?; + if let Some(ramdisk) = images.ramdisk { + self.load_ramdisk_from_memory(ramdisk)?; + } + if let Some(dtb) = self.guest_dtb.as_ref() { + load_guest_dtb(self, dtb)?; + } + self.load_boot_image_from_memory(images.bios) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + pub(crate) fn load_standard_images_from_filesystem( + &mut self, + load_guest_dtb: fn(&Self, &crate::boot::fdt::GuestDtbImage) -> AxResult, + ) -> AxResult { + fs::load_vm_image( + &self.config.kernel.kernel_path, + self.kernel_load_gpa, + self.vm.clone(), + self.provider, + )?; + self.load_boot_image_from_filesystem()?; + if let Some(ramdisk_path) = &self.config.kernel.ramdisk_path { + self.load_ramdisk_from_filesystem(ramdisk_path)?; + } + if let Some(dtb) = self.guest_dtb.as_ref() { + load_guest_dtb(self, dtb)?; + } + Ok(()) + } + + pub(crate) fn load_ramdisk_from_memory(&self, ramdisk: &[u8]) -> AxResult { + let load_gpa = self.ramdisk_load_gpa()?; + self.record_ramdisk_size(ramdisk.len()); + info!( + "Loading ramdisk image from memory ({} bytes) into GPA @{:#x}", + ramdisk.len(), + load_gpa.as_usize() + ); + load_vm_image_from_memory(ramdisk, load_gpa, self.vm.clone()) + } + + pub(crate) fn ramdisk_load_gpa(&self) -> AxResult { + self.ramdisk_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "Ramdisk load addr is missed")) + } + + fn capture_prepared_load_addresses(&mut self) { + self.vm.with_config(|config| { + self.kernel_load_gpa = config.image_config.kernel_load_gpa; + self.bios_load_gpa = config.image_config.bios_load_gpa; + self.ramdisk_load_gpa = config.image_config.ramdisk.as_ref().map(|r| r.load_gpa); + }); + } + + fn load_boot_image_from_memory(&self, bios: Option<&[u8]>) -> AxResult { + if !self.config.kernel.enable_bios { + return Ok(()); + } + let Some(bios) = bios else { + return Ok(()); + }; + let load_gpa = self + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; + load_vm_image_from_memory(bios, load_gpa, self.vm.clone()) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + fn load_boot_image_from_filesystem(&self) -> AxResult { + if !self.config.kernel.enable_bios { + return Ok(()); + } + let Some(path) = self.config.kernel.boot_firmware_path() else { + return Ok(()); + }; + let load_gpa = self + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; + fs::load_vm_image(path, load_gpa, self.vm.clone(), self.provider) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + pub(crate) fn load_ramdisk_from_filesystem(&self, ramdisk_path: &str) -> AxResult { + let load_gpa = self.ramdisk_load_gpa()?; + let ramdisk_size = fs::image_size(ramdisk_path, self.provider)?; + self.record_ramdisk_size(ramdisk_size); + info!( + "Loading ramdisk image from filesystem {} ({} bytes) into GPA @{:#x}", + ramdisk_path, + ramdisk_size, + load_gpa.as_usize() + ); + fs::load_vm_image(ramdisk_path, load_gpa, self.vm.clone(), self.provider) + } + + fn record_ramdisk_size(&self, size: usize) { + self.vm.with_config(|config| { + if let Some(ramdisk) = config.image_config.ramdisk.as_mut() { + ramdisk.size = Some(size); + } + }); + } +} + +fn with_memory_image( + config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, + func: F, +) -> Option +where + F: FnOnce(&[u8]) -> R, +{ + provider + .static_vm_images() + .iter() + .find(|image| image.id == config.base.id) + .map(|image| func(image.kernel)) +} + +fn memory_images_for_vm( + config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult { + provider + .static_vm_images() + .iter() + .copied() + .find(|image| image.id == config.base.id) + .ok_or_else(|| { + ax_err_type!( + NotFound, + "VM images are missing; pass VM configs with AXVISOR_VM_CONFIGS" + ) + }) +} + +pub fn load_vm_image_from_memory( + image_buffer: &[u8], + load_addr: GuestPhysAddr, + vm: AxVMRef, +) -> AxResult { + let mut buffer_pos = 0; + let image_size = image_buffer.len(); + let image_load_regions = vm.get_image_load_region(load_addr, image_size)?; + + for region in image_load_regions { + let bytes_to_write = region.len().min(image_size - buffer_pos); + // SAFETY: The destination comes from `get_image_load_region`, the source + // contains `bytes_to_write` bytes, and guest memory cannot overlap the image. + unsafe { + core::ptr::copy_nonoverlapping( + image_buffer[buffer_pos..].as_ptr(), + region.as_mut_ptr().cast(), + bytes_to_write, + ); + } + crate::clean_dcache_range((region.as_ptr() as usize).into(), bytes_to_write); + buffer_pos += bytes_to_write; + if buffer_pos == image_size { + break; + } + } + + if buffer_pos == image_size { + Ok(()) + } else { + ax_err!( + InvalidData, + format!("VM image was only partially loaded: {buffer_pos}/{image_size} bytes") + ) + } +} + +#[cfg(any(feature = "fs", feature = "host-fs"))] +pub mod fs { + use alloc::{format, vec::Vec}; + + use ax_errno::{AxResult, ax_err_type}; + use axvmconfig::AxVMCrateConfig; + + use crate::{AxVMRef, GuestPhysAddr, boot::BootImageProvider}; + + pub fn kernel_read( + config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, + read_size: usize, + ) -> AxResult> { + provider.read_file_exact(&config.kernel.kernel_path, read_size) + } + + pub(crate) fn load_vm_image( + image_path: &str, + image_load_gpa: GuestPhysAddr, + vm: AxVMRef, + provider: &dyn BootImageProvider, + ) -> AxResult { + let image = provider.read_file(image_path)?; + let image_load_regions = vm.get_image_load_region(image_load_gpa, image.len())?; + let mut offset = 0; + for buffer in image_load_regions { + let end = offset + buffer.len(); + let data = image.get(offset..end).ok_or_else(|| { + ax_err_type!( + InvalidData, + format!("Image {image_path} has an invalid load region layout") + ) + })?; + buffer.copy_from_slice(data); + offset = end; + crate::clean_dcache_range((buffer.as_ptr() as usize).into(), buffer.len()); + } + Ok(()) + } + + pub fn image_size(file_name: &str, provider: &dyn BootImageProvider) -> AxResult { + provider.file_size(file_name) + } + + pub fn read_full_image(file_name: &str, provider: &dyn BootImageProvider) -> AxResult> { + provider.read_file(file_name) + } +} diff --git a/virtualization/axvm/src/boot/mod.rs b/virtualization/axvm/src/boot/mod.rs index 146404050e..44237fa56e 100644 --- a/virtualization/axvm/src/boot/mod.rs +++ b/virtualization/axvm/src/boot/mod.rs @@ -3,12 +3,14 @@ #[cfg(any(feature = "fs", feature = "host-fs"))] use ax_errno::ax_err_type; +pub mod fdt; +pub mod guest_platform; +pub mod images; mod policy; + pub use images::*; pub use policy::{GuestAcpiTables, GuestBootDescription, GuestDeviceTree, GuestFdtBuilder}; -pub use crate::arch::{fdt, guest_platform, images}; - /// Build-time image bytes supplied by the hypervisor application. #[derive(Clone, Copy, Debug)] pub struct StaticVmImage { diff --git a/virtualization/axvm/src/host/arceos.rs b/virtualization/axvm/src/host/arceos.rs index 1ebd60ffb6..8fd7464d5b 100644 --- a/virtualization/axvm/src/host/arceos.rs +++ b/virtualization/axvm/src/host/arceos.rs @@ -1,3 +1,262 @@ -//! Re-export the architecture-owned ArceOS host adapter for common runtime code. +//! Default private ArceOS host adapter for AxVM. -pub(crate) use crate::arch::arceos::*; +extern crate alloc; + +use alloc::boxed::Box; +use core::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; + +use ax_errno::AxResult; +use ax_memory_addr::PAGE_SIZE_4K; +use ax_std::{ + os::arceos::{api, modules}, + thread, +}; +use axvm_types::{HostPhysAddr, HostVirtAddr}; + +use crate::{ + arch::{ArchOps, CurrentArch}, + host::{HostCpu, HostMemory, HostPlatform, HostTime}, +}; + +/// Private default host adapter used by [`crate::AxvmRuntime`]. +pub(crate) struct ArceOsHost; + +const CPU_ENABLE_WAIT_TIMEOUT: Duration = Duration::from_secs(5); + +static ARCEOS_HOST: ArceOsHost = ArceOsHost; + +pub(crate) fn arceos_host() -> &'static ArceOsHost { + &ARCEOS_HOST +} + +impl HostMemory for ArceOsHost { + fn alloc_frame(&self) -> Option { + modules::ax_alloc::global_allocator() + .alloc_pages(1, PAGE_SIZE_4K, modules::ax_alloc::UsageKind::PageTable) + .map(|vaddr| self.virt_to_phys(vaddr.into())) + .ok() + } + + fn dealloc_frame(&self, paddr: HostPhysAddr) { + modules::ax_alloc::global_allocator().dealloc_pages( + self.phys_to_virt(paddr).as_usize(), + 1, + modules::ax_alloc::UsageKind::PageTable, + ); + } + + fn alloc_contiguous_frames( + &self, + num_frames: usize, + frame_align: usize, + ) -> Option { + modules::ax_alloc::global_allocator() + .alloc_pages( + num_frames, + frame_align.max(PAGE_SIZE_4K), + modules::ax_alloc::UsageKind::Dma, + ) + .map(|vaddr| self.virt_to_phys(vaddr.into())) + .ok() + } + + fn dealloc_contiguous_frames(&self, paddr: HostPhysAddr, num_frames: usize) { + modules::ax_alloc::global_allocator().dealloc_pages( + self.phys_to_virt(paddr).as_usize(), + num_frames, + modules::ax_alloc::UsageKind::Dma, + ); + } + + fn phys_to_virt(&self, paddr: HostPhysAddr) -> HostVirtAddr { + modules::ax_hal::mem::phys_to_virt(paddr) + } + + fn virt_to_phys(&self, vaddr: HostVirtAddr) -> HostPhysAddr { + modules::ax_hal::mem::virt_to_phys(vaddr) + } +} + +impl HostTime for ArceOsHost { + type CancelToken = usize; + + fn monotonic_time(&self) -> Duration { + modules::ax_hal::time::monotonic_time() + } + + fn set_oneshot_timer(&self, deadline_ns: u64) { + crate::arch::set_oneshot_timer(deadline_ns); + } + + fn register_timer( + &self, + deadline_ns: u64, + callback: Box, + ) -> Self::CancelToken { + crate::timer::register_timer(deadline_ns, callback) + } +} + +pub(crate) fn dispatch_host_irq(vector: usize) { + modules::ax_hal::irq::handle_irq(vector); +} + +impl HostCpu for ArceOsHost { + type CpuMask = api::task::AxCpuMask; + + fn cpu_count(&self) -> usize { + modules::ax_hal::cpu_num() + } + + fn this_cpu_id(&self) -> usize { + modules::ax_hal::percpu::this_cpu_id() + } +} + +pub(crate) fn cpu_mask_from_raw_bits(bits: usize) -> api::task::AxCpuMask { + api::task::AxCpuMask::from_raw_bits(bits) +} + +pub(crate) type ArceOsCpuMask = api::task::AxCpuMask; +pub(crate) type ArceOsAxTaskExt = modules::ax_task::AxTaskExt; +pub(crate) type ArceOsAxTaskRef = modules::ax_task::AxTaskRef; +pub(crate) type ArceOsCurrentTask = modules::ax_task::CurrentTask; +pub(crate) type ArceOsTaskInner = modules::ax_task::TaskInner; +pub(crate) type ArceOsWaitQueue = modules::ax_task::WaitQueue; +pub(crate) type ArceOsWaitQueueHandle = api::task::AxWaitQueueHandle; +pub(crate) use modules::ax_task::TaskExt as ArceOsTaskExt; + +pub(crate) fn current_task() -> ArceOsCurrentTask { + modules::ax_task::current() +} + +pub(crate) fn spawn_task(task: ArceOsTaskInner) -> ArceOsAxTaskRef { + modules::ax_task::spawn_task(task) +} + +pub(crate) fn yield_now() { + thread::yield_now(); +} + +pub(crate) fn wait_queue_wait_until( + queue: &api::task::AxWaitQueueHandle, + condition: impl Fn() -> bool, +) { + api::task::ax_wait_queue_wait_until(queue, condition, None); +} + +pub(crate) fn wait_queue_wake(queue: &api::task::AxWaitQueueHandle, count: u32) { + api::task::ax_wait_queue_wake(queue, count); +} + +pub(crate) fn send_ipi(cpu_id: usize) { + if modules::ax_hal::percpu::this_cpu_id() == cpu_id { + return; + } + modules::ax_hal::irq::send_ipi( + modules::ax_hal::irq::ipi_irq(), + modules::ax_hal::irq::IpiTarget::Other { cpu_id }, + ); +} + +fn send_ipi_to_all_except_current(cpu_num: usize) { + if cpu_num <= 1 { + return; + } + let cpu_id = modules::ax_hal::percpu::this_cpu_id(); + modules::ax_hal::irq::send_ipi( + modules::ax_hal::irq::ipi_irq(), + modules::ax_hal::irq::IpiTarget::AllExceptCurrent { cpu_id, cpu_num }, + ); +} + +#[cfg(any(feature = "fs", feature = "host-fs"))] +pub fn shutdown_host_filesystems() -> AxResult { + modules::ax_fs_ng::shutdown_filesystems()?; + let released = modules::ax_fs_ng::release_block_irqs_for_passthrough(); + if released != 0 { + info!("Released {released} host filesystem block IRQ registration(s) before passthrough"); + } + Ok(()) +} + +impl HostPlatform for ArceOsHost { + fn has_hardware_support(&self) -> bool { + CurrentArch::has_hardware_support() + } + + fn enable_virtualization_on_current_cpu(&self) -> AxResult { + crate::timer::init_percpu(); + crate::percpu::init_current_cpu()?; + crate::percpu::enable_current_cpu()?; + crate::percpu::mark_cpu_enabled(self.this_cpu_id()); + Ok(()) + } + + fn enable_virtualization_on_all_cpus(&self) -> AxResult { + static CORES: AtomicUsize = AtomicUsize::new(0); + + info!("Enabling hardware virtualization support on all cores..."); + CORES.store(0, Ordering::Release); + crate::percpu::reset_enabled_cpu_mask(); + + let cpu_count = self.cpu_count(); + let current_cpu = self.this_cpu_id(); + info!("Core {current_cpu} is initializing hardware virtualization support..."); + self.enable_virtualization_on_current_cpu()?; + info!("Hardware virtualization support enabled on core {current_cpu}"); + CORES.store(1, Ordering::Release); + + for cpu_id in 0..cpu_count { + if cpu_id == current_cpu { + continue; + } + let task = modules::ax_task::TaskInner::new( + move || { + let host = arceos_host(); + info!("Core {cpu_id} is initializing hardware virtualization support..."); + host.enable_virtualization_on_current_cpu() + .expect("failed to enable hardware virtualization"); + info!("Hardware virtualization support enabled on core {cpu_id}"); + let _ = CORES.fetch_add(1, Ordering::Release); + }, + alloc::format!("axvm-hv-init-{cpu_id}"), + modules::ax_task::default_task_stack_size(), + ); + task.set_cpumask(::CpuMask::one_shot(cpu_id)); + modules::ax_task::spawn_task(task); + if cpu_id != self.this_cpu_id() { + send_ipi(cpu_id); + } + } + + info!("Waiting for all cores to enable hardware virtualization..."); + let start = self.monotonic_time(); + let mut wait_rounds = 0usize; + while CORES.load(Ordering::Acquire) != cpu_count { + thread::yield_now(); + wait_rounds = wait_rounds.wrapping_add(1); + if wait_rounds.is_multiple_of(256) { + send_ipi_to_all_except_current(cpu_count); + } + if self.monotonic_time().saturating_sub(start) >= CPU_ENABLE_WAIT_TIMEOUT { + break; + } + } + CurrentArch::register_platform_irq_injector(); + let enabled_count = CORES.load(Ordering::Acquire); + if enabled_count == cpu_count { + info!("All cores have enabled hardware virtualization support."); + } else { + warn!( + "Only {enabled_count}/{cpu_count} cores enabled hardware virtualization before \ + timeout; continuing with host CPU mask {:#x}", + crate::percpu::enabled_cpu_mask() + ); + } + Ok(()) + } +} diff --git a/virtualization/axvm/src/host/mod.rs b/virtualization/axvm/src/host/mod.rs index deb8952786..f3003f1839 100644 --- a/virtualization/axvm/src/host/mod.rs +++ b/virtualization/axvm/src/host/mod.rs @@ -1,7 +1,6 @@ //! Internal host boundary used by the AxVM runtime. pub(crate) mod arceos; -pub(crate) mod irq; pub(crate) mod paging; pub(crate) mod task; pub(crate) mod traits; @@ -11,4 +10,4 @@ pub(crate) fn default_host() -> &'static arceos::ArceOsHost { } pub(crate) use paging::PagingHandler; -pub(crate) use traits::{HostConsole, HostCpu, HostMemory, HostPlatform, HostTime}; +pub(crate) use traits::{HostCpu, HostMemory, HostPlatform, HostTime}; diff --git a/virtualization/axvm/src/host/traits.rs b/virtualization/axvm/src/host/traits.rs index ba8e9d43d8..54c6582143 100644 --- a/virtualization/axvm/src/host/traits.rs +++ b/virtualization/axvm/src/host/traits.rs @@ -38,9 +38,6 @@ pub trait HostTime { /// Timer cancellation token. type CancelToken: Copy + Send + Sync + 'static; - /// Convert nanoseconds to hardware ticks. - fn nanos_to_ticks(&self, nanos: u64) -> u64; - /// Read monotonic host time. fn monotonic_time(&self) -> Duration; @@ -53,9 +50,6 @@ pub trait HostTime { deadline_ns: u64, callback: Box, ) -> Self::CancelToken; - - /// Cancel a VM timer callback. - fn cancel_timer(&self, token: Self::CancelToken); } /// Host CPU topology and affinity operations. @@ -70,15 +64,6 @@ pub trait HostCpu { fn this_cpu_id(&self) -> usize; } -/// Host console operations. -pub trait HostConsole { - /// Write raw bytes to host console. - fn write_bytes(&self, bytes: &[u8]); - - /// Read raw bytes from host console. - fn read_bytes(&self, bytes: &mut [u8]) -> usize; -} - /// Host platform lifecycle and virtualization controls. pub trait HostPlatform { /// Check whether hardware virtualization is available. diff --git a/virtualization/axvm/src/lib.rs b/virtualization/axvm/src/lib.rs index 28decb2270..d1164e50d2 100644 --- a/virtualization/axvm/src/lib.rs +++ b/virtualization/axvm/src/lib.rs @@ -24,12 +24,14 @@ extern crate alloc; extern crate log; mod arch; +mod architecture; pub mod boot; mod host; pub mod irq; pub mod layout; pub mod lifecycle; mod manager; +mod npt; mod percpu; mod runtime; mod task; diff --git a/virtualization/axvm/src/arch/npt.rs b/virtualization/axvm/src/npt.rs similarity index 100% rename from virtualization/axvm/src/arch/npt.rs rename to virtualization/axvm/src/npt.rs diff --git a/virtualization/axvm/src/runtime/vcpus.rs b/virtualization/axvm/src/runtime/vcpus.rs index 8466290283..dad9b3d24c 100644 --- a/virtualization/axvm/src/runtime/vcpus.rs +++ b/virtualization/axvm/src/runtime/vcpus.rs @@ -315,14 +315,20 @@ fn vcpu_run() { CurrentArch::before_vcpu_run(&vm, &vcpu); match CurrentArch::run_vcpu(&vm, &vcpu) { - Ok(VcpuRunAction::Yield) => {} - Ok(VcpuRunAction::Wait) => wait(&runtime), - Ok(VcpuRunAction::Stop(reason)) => { + Ok(VcpuRunAction { + stop_reason: Some(reason), + .. + }) => { if let Err(err) = vm.stop(reason) { warn!("VM[{vm_id}] shutdown failed: {err:?}"); } notify_all_vcpus(vm_id); } + Ok(VcpuRunAction { + waits_for_event: true, + .. + }) => wait(&runtime), + Ok(VcpuRunAction { .. }) => {} Err(err) => { error!("VM[{vm_id}] run VCpu[{vcpu_id}] get error {err:?}"); if let Err(err) = vm.stop(StopReason::Fault(format!("{err:?}"))) { diff --git a/virtualization/axvm/tests/arch_boundary_contract.rs b/virtualization/axvm/tests/arch_boundary_contract.rs index b1ff37c57e..04653d34aa 100644 --- a/virtualization/axvm/tests/arch_boundary_contract.rs +++ b/virtualization/axvm/tests/arch_boundary_contract.rs @@ -59,7 +59,7 @@ fn production_sources_keep_architecture_cfg_inside_arch_module() { } #[test] -fn arch_root_contains_only_architectures_and_shared_dispatch_modules() { +fn arch_root_contains_only_architecture_directories_and_dispatch_page() { let arch_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/arch"); let mut unexpected_entries = std::fs::read_dir(&arch_root) .expect("AxVM architecture directory must be readable") @@ -68,7 +68,7 @@ fn arch_root_contains_only_architectures_and_shared_dispatch_modules() { .filter(|name| { !matches!( name.as_str(), - "aarch64" | "loongarch64" | "riscv64" | "x86_64" | "mod.rs" | "npt.rs" + "aarch64" | "loongarch64" | "riscv64" | "x86_64" | "mod.rs" ) }) .collect::>(); @@ -76,12 +76,46 @@ fn arch_root_contains_only_architectures_and_shared_dispatch_modules() { assert!( unexpected_entries.is_empty(), - "arch root must contain only architecture directories and shared dispatch modules; found: \ - {}", + "arch root must contain only architecture directories and the dispatch page; found: {}", unexpected_entries.join(", ") ); } +#[test] +fn arch_dispatch_page_does_not_own_common_implementations() { + let dispatch = include_str!("../src/arch/mod.rs"); + + for forbidden in [ + "#[path", + "trait ArchOps", + "struct MmioReadExit", + "fn handle_mmio_read", + "fn default_vcpu_affinities", + ] { + assert!( + !dispatch.contains(forbidden), + "arch/mod.rs must only select and export the current architecture: {forbidden}" + ); + } +} + +#[test] +fn common_domains_live_outside_architecture_directories() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + + for relative_path in [ + "boot/fdt/mod.rs", + "boot/images/mod.rs", + "host/arceos.rs", + "npt.rs", + ] { + assert!( + source_root.join(relative_path).is_file(), + "common AxVM domain must use its canonical source path: {relative_path}" + ); + } +} + fn find_target_arch_cfg_outside_arch( source_root: &std::path::Path, directory: &std::path::Path, From 1bb69c2b19e2ad411ed10a813f9e183c53abfc6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Fri, 10 Jul 2026 15:13:40 +0800 Subject: [PATCH 3/5] refactor(axvm): unify guest boot preparation --- os/axvisor/src/config.rs | 98 +------ .../axvm/src/arch/aarch64/boot/fdt/mod.rs | 257 ------------------ .../axvm/src/arch/aarch64/capabilities.rs | 60 +++- virtualization/axvm/src/arch/aarch64/fdt.rs | 97 +++++++ virtualization/axvm/src/arch/aarch64/mod.rs | 3 +- .../axvm/src/arch/loongarch64/capabilities.rs | 25 +- .../axvm/src/arch/loongarch64/fdt.rs | 24 ++ .../axvm/src/arch/loongarch64/mod.rs | 3 +- virtualization/axvm/src/arch/mod.rs | 28 +- .../axvm/src/arch/riscv64/capabilities.rs | 62 ++++- virtualization/axvm/src/arch/riscv64/fdt.rs | 61 +++++ virtualization/axvm/src/arch/riscv64/mod.rs | 1 + .../axvm/src/arch/x86_64/boot/mod.rs | 10 + .../axvm/src/arch/x86_64/capabilities.rs | 6 +- virtualization/axvm/src/arch/x86_64/fdt.rs | 4 + virtualization/axvm/src/arch/x86_64/mod.rs | 5 +- .../axvm/src/architecture/capabilities.rs | 19 ++ virtualization/axvm/src/architecture/mod.rs | 2 +- .../boot/fdt => boot/fdt/core}/create.rs | 150 ++-------- .../axvm/src/boot/fdt/{ => core}/device.rs | 0 virtualization/axvm/src/boot/fdt/core/mod.rs | 174 ++++++++++++ .../boot/fdt => boot/fdt/core}/parser.rs | 79 +----- .../axvm/src/boot/fdt/core/policy.rs | 17 ++ .../axvm/src/boot/fdt/{ => core}/print.rs | 0 .../axvm/src/boot/fdt/{ => core}/tree.rs | 0 .../src/boot/fdt/{ => core}/tree_tests.rs | 0 virtualization/axvm/src/boot/fdt/mod.rs | 66 ++++- virtualization/axvm/src/boot/mod.rs | 12 +- virtualization/axvm/src/boot/policy.rs | 29 ++ virtualization/axvm/src/boot/prepared.rs | 53 ++++ virtualization/axvm/src/host/arceos.rs | 11 - virtualization/axvm/src/host/traits.rs | 13 - .../axvm/tests/arch_boundary_contract.rs | 102 +++++++ 33 files changed, 887 insertions(+), 584 deletions(-) delete mode 100644 virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs create mode 100644 virtualization/axvm/src/arch/aarch64/fdt.rs create mode 100644 virtualization/axvm/src/arch/loongarch64/fdt.rs create mode 100644 virtualization/axvm/src/arch/riscv64/fdt.rs create mode 100644 virtualization/axvm/src/arch/x86_64/fdt.rs rename virtualization/axvm/src/{arch/aarch64/boot/fdt => boot/fdt/core}/create.rs (71%) rename virtualization/axvm/src/boot/fdt/{ => core}/device.rs (100%) create mode 100644 virtualization/axvm/src/boot/fdt/core/mod.rs rename virtualization/axvm/src/{arch/aarch64/boot/fdt => boot/fdt/core}/parser.rs (90%) create mode 100644 virtualization/axvm/src/boot/fdt/core/policy.rs rename virtualization/axvm/src/boot/fdt/{ => core}/print.rs (100%) rename virtualization/axvm/src/boot/fdt/{ => core}/tree.rs (100%) rename virtualization/axvm/src/boot/fdt/{ => core}/tree_tests.rs (100%) create mode 100644 virtualization/axvm/src/boot/prepared.rs diff --git a/os/axvisor/src/config.rs b/os/axvisor/src/config.rs index dbd1996e09..619da69ef5 100644 --- a/os/axvisor/src/config.rs +++ b/os/axvisor/src/config.rs @@ -22,11 +22,12 @@ use core::sync::atomic::{AtomicBool, Ordering}; use ax_errno::{AxResult, ax_err_type}; #[cfg(all(feature = "fs", target_arch = "x86_64"))] use axvm::InterruptTriggerMode; -#[cfg(target_arch = "x86_64")] -use axvm::config::VMBootProtocol; use axvm::{ AxVM, GuestPhysAddr, - boot::{BootImageProvider, ImageLoader, StaticVmImage, get_image_header}, + boot::{ + BootImageProvider, StaticVmImage, boot_firmware_load_gpa, get_image_header, + guest_boot_policy, init_guest_boot_resources, prepare_guest_boot, + }, config::{ AxVCpuConfig, AxVMConfig, AxVMConfigParams, GuestBootPolicy, PhysCpuList, RamdiskInfo, VMImageConfig, @@ -34,17 +35,6 @@ use axvm::{ }; use axvmconfig::{AxVMCrateConfig, VMType}; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use axvm::boot::fdt::handle_fdt_operations; -#[cfg(target_arch = "loongarch64")] -use axvm::boot::fdt::{handle_fdt_operations, init_guest_boot_resources}; -#[cfg(target_arch = "x86_64")] -use axvm::boot::is_x86_linux_image_config; - -/// Default BIOS load GPA for x86_64 built-in BIOS. -#[cfg(target_arch = "x86_64")] -const DEFAULT_X86_BIOS_LOAD_GPA: usize = 0x8000; - #[cfg(all( feature = "fs", any(target_arch = "x86_64", target_arch = "loongarch64") @@ -88,11 +78,7 @@ pub mod vmcfg { } pub fn init_guest_vms() { - // Initialize LoongArch firmware resources before guest configs are materialized. - #[cfg(target_arch = "loongarch64")] - { - init_guest_boot_resources(); - } + init_guest_boot_resources(); // First try to get configs from filesystem if fs feature is enabled let mut gvm_raw_configs = vmcfg::filesystem_vm_configs(); @@ -120,8 +106,7 @@ pub fn init_guest_vms() { pub fn init_guest_vm(raw_cfg: &str) -> AxResult { let image_provider = AxvisorBootImageProvider; - #[allow(unused_mut)] - let mut vm_create_config = AxVMCrateConfig::from_toml(raw_cfg) + let vm_create_config = AxVMCrateConfig::from_toml(raw_cfg) .map_err(|e| ax_err_type!(InvalidData, format!("Failed to resolve VM config: {e:?}")))?; #[cfg(all( @@ -137,25 +122,13 @@ pub fn init_guest_vm(raw_cfg: &str) -> AxResult { ); } - #[allow(unused_mut)] let mut vm_config = build_axvm_config(&vm_create_config); + let prepared_boot = prepare_guest_boot(&mut vm_config, vm_create_config, &image_provider)?; + let prepared_config = prepared_boot.config(); - // Handle FDT-related operations for architectures that boot guests with DTB. - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - let guest_dtb = handle_fdt_operations(&mut vm_config, &mut vm_create_config, &image_provider)?; - #[cfg(target_arch = "loongarch64")] - handle_fdt_operations(&mut vm_config, &mut vm_create_config)?; + sync_axvm_config_from_crate_config(&mut vm_config, prepared_config); - sync_axvm_config_from_crate_config(&mut vm_config, &vm_create_config); - - #[cfg(target_arch = "x86_64")] - let skip_guest_address_adjustment = x86_linux_direct_boot_config(&vm_create_config); - #[cfg(not(target_arch = "x86_64"))] - let skip_guest_address_adjustment = false; - vm_config.set_boot_policy(guest_boot_policy( - &vm_create_config, - skip_guest_address_adjustment, - )); + vm_config.set_boot_policy(guest_boot_policy(prepared_config, &image_provider)); // info!("after parse_vm_interrupt, crate VM[{}] with config: {:#?}", vm_config.id(), vm_config); info!("Creating VM[{}] {:?}", vm_config.id(), vm_config.name()); @@ -171,17 +144,7 @@ pub fn init_guest_vm(raw_cfg: &str) -> AxResult { // Load corresponding images for VM. info!("VM[{}] created success, loading images...", vm.id()); - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - let mut loader = ImageLoader::new( - main_mem, - vm_create_config, - vm.clone(), - &image_provider, - guest_dtb, - ); - #[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64")))] - let mut loader = ImageLoader::new(main_mem, vm_create_config, vm.clone(), &image_provider); - loader.load()?; + prepared_boot.load_images(main_mem, vm.clone(), &image_provider)?; vm.prepare() .map_err(|e| ax_err_type!(InvalidData, format!("VM[{}] setup failed: {e:?}", vm.id())))?; @@ -225,7 +188,7 @@ pub(crate) fn build_axvm_config(cfg: &AxVMCrateConfig) -> AxVMConfig { image_config: VMImageConfig { kernel_load_gpa: GuestPhysAddr::from(cfg.kernel.kernel_load_addr), loaded_from_filesystem: cfg.kernel.image_location.as_deref() == Some("fs"), - bios_load_gpa: configured_bios_load_gpa(cfg), + bios_load_gpa: boot_firmware_load_gpa(cfg), dtb_load_gpa: cfg.kernel.dtb_load_addr.map(GuestPhysAddr::from), ramdisk: cfg.kernel.ramdisk_load_addr.map(|addr| RamdiskInfo { load_gpa: GuestPhysAddr::from(addr), @@ -249,38 +212,6 @@ fn sync_axvm_config_from_crate_config(vm_config: &mut AxVMConfig, cfg: &AxVMCrat vm_config.set_memory_regions(cfg.kernel.memory_regions.clone()); } -fn guest_boot_policy( - cfg: &AxVMCrateConfig, - skip_guest_address_adjustment: bool, -) -> GuestBootPolicy { - if skip_guest_address_adjustment { - GuestBootPolicy::KeepConfigured - } else { - GuestBootPolicy::AdjustKernelForBootProtocol { - protocol: cfg.kernel.effective_boot_protocol(), - } - } -} - -fn configured_bios_load_gpa(cfg: &AxVMCrateConfig) -> Option { - if !cfg.kernel.enable_bios { - return None; - } - - if let Some(addr) = cfg.kernel.bios_load_addr { - return Some(GuestPhysAddr::from(addr)); - } - - #[cfg(target_arch = "x86_64")] - if cfg.kernel.boot_firmware_path().is_none() - && cfg.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot - { - return Some(GuestPhysAddr::from(DEFAULT_X86_BIOS_LOAD_GPA)); - } - - None -} - #[cfg(all( feature = "fs", any(target_arch = "x86_64", target_arch = "loongarch64") @@ -419,11 +350,6 @@ fn x86_intx_forwarding_trigger(binding: &ax_driver::BindingIrq) -> InterruptTrig } } -#[cfg(target_arch = "x86_64")] -fn x86_linux_direct_boot_config(config: &AxVMCrateConfig) -> bool { - is_x86_linux_image_config(config, &AxvisorBootImageProvider) -} - struct AxvisorBootImageProvider; impl BootImageProvider for AxvisorBootImageProvider { diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs b/virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs deleted file mode 100644 index 396c26fac5..0000000000 --- a/virtualization/axvm/src/arch/aarch64/boot/fdt/mod.rs +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright 2025 The Axvisor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! FDT (Flattened Device Tree) processing module for AxVisor. -//! -//! This module provides functionality for parsing and processing device tree blobs, -//! including CPU configuration, passthrough device detection, and FDT generation. - -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -mod create; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -#[path = "../../../../boot/fdt/device.rs"] -mod device; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -mod parser; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -#[path = "../../../../boot/fdt/print.rs"] -mod print; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -#[path = "../../../../boot/fdt/tree.rs"] -mod tree; - -#[cfg(test)] -#[path = "../../../../boot/fdt/tree_tests.rs"] -mod tree_tests; - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use alloc::format; -use alloc::vec::Vec; - -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -use ax_errno::AxResult; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use ax_errno::ax_err_type; -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -use axvmconfig::{AxVMCrateConfig, VMBootProtocol}; -// pub use print::print_fdt; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -pub use create::update_fdt; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -pub use parser::*; - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use crate::boot::BootImageProvider; -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -use crate::config::AxVMConfig; - -/// Guest DTB artifact produced or patched by the monitor before AxVM owns it. -#[derive(Debug, Clone)] -pub struct GuestDtbImage { - bytes: Vec, -} - -impl GuestDtbImage { - pub fn new(bytes: Vec) -> Self { - Self { bytes } - } - - pub fn as_bytes(&self) -> &[u8] { - &self.bytes - } -} - -/// Initialize LoongArch guest firmware resource handling. -#[cfg(target_arch = "loongarch64")] -pub fn init_guest_boot_resources() { - crate::arch::guest_platform::init(); -} - -#[cfg(target_arch = "loongarch64")] -fn handle_uefi_fdt_operations( - vm_config: &mut AxVMConfig, - vm_create_config: &mut AxVMCrateConfig, -) -> AxResult { - crate::arch::guest_platform::prepare_uefi_fdt_config(vm_config, vm_create_config) -} - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -fn handle_uefi_fdt_operations( - vm_config: &mut AxVMConfig, - vm_create_config: &mut AxVMCrateConfig, -) -> AxResult { - info!( - "VM[{}] uses UEFI boot protocol, skipping guest DTB handling", - vm_config.id() - ); - vm_config.clear_dtb_load_gpa(); - vm_create_config.kernel.dtb_load_addr = None; - Ok(()) -} - -/// Prepare LoongArch guest firmware FDT facts for UEFI boot. -#[cfg(target_arch = "loongarch64")] -pub fn handle_fdt_operations( - vm_config: &mut AxVMConfig, - vm_create_config: &mut AxVMCrateConfig, -) -> AxResult { - if vm_create_config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { - handle_uefi_fdt_operations(vm_config, vm_create_config)?; - return Ok(()); - } - - ax_errno::ax_err!( - Unsupported, - "LoongArch AxVisor guests currently require UEFI boot" - ) -} - -/// Handle all FDT-related operations for guest architectures that boot with DTB. -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -pub fn handle_fdt_operations( - vm_config: &mut AxVMConfig, - vm_create_config: &mut AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> AxResult> { - if vm_create_config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { - handle_uefi_fdt_operations(vm_config, vm_create_config)?; - return Ok(None); - } - - let host_fdt_bytes = try_get_host_fdt(); - let mut guest_dtb = None; - - if let Some(host_fdt_bytes) = host_fdt_bytes { - let host_fdt = fdt_edit::Fdt::from_bytes(host_fdt_bytes) - .map_err(|e| ax_err_type!(InvalidData, format!("Failed to parse host FDT: {e:#?}")))?; - set_phys_cpu_sets(vm_config, &host_fdt, vm_create_config)?; - - if let Some(provided_dtb) = - get_developer_provided_dtb(vm_config, vm_create_config, provider)? - { - info!("VM[{}] found DTB , parsing...", vm_config.id()); - reserve_excluded_device_ranges(vm_config, vm_create_config, &provided_dtb)?; - guest_dtb = Some(GuestDtbImage::new(update_provided_fdt( - &provided_dtb, - Some(host_fdt_bytes), - vm_create_config, - )?)); - } else { - info!( - "VM[{}] DTB not found, generating based on the configuration file.", - vm_config.id() - ); - guest_dtb = Some(GuestDtbImage::new(setup_guest_fdt_from_vmm( - host_fdt_bytes, - vm_config, - vm_create_config, - )?)); - } - } else if let Some(provided_dtb) = - get_developer_provided_dtb(vm_config, vm_create_config, provider)? - { - info!("VM[{}] found DTB , parsing...", vm_config.id()); - reserve_excluded_device_ranges(vm_config, vm_create_config, &provided_dtb)?; - guest_dtb = Some(GuestDtbImage::new(update_provided_fdt( - &provided_dtb, - None, - vm_create_config, - )?)); - } else { - warn!( - "VM[{}] no guest DTB provided; continuing without generated DTB", - vm_config.id() - ); - } - - // Overlay VM config with the given DTB. - if let Some(dtb) = guest_dtb.as_ref().map(GuestDtbImage::as_bytes) { - parse_reserved_memory_regions(vm_create_config, dtb)?; - parse_passthrough_devices_address(vm_config, vm_create_config, dtb)?; - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - parse_vm_interrupt(vm_config, dtb)?; - } else { - error!( - "VM[{}] DTB not found in memory, skipping...", - vm_config.id() - ); - let unresolved_passthrough_devices = vm_config - .pass_through_devices() - .iter() - .filter(|device| device.length == 0) - .cloned() - .collect::>(); - if !unresolved_passthrough_devices.is_empty() { - warn!( - "VM[{}] clearing {} unresolved passthrough discovery device(s)", - vm_config.id(), - unresolved_passthrough_devices.len() - ); - for device in unresolved_passthrough_devices { - vm_config.remove_pass_through_device(device); - } - } - vm_config.clear_dtb_load_gpa(); - vm_create_config.kernel.dtb_load_addr = None; - } - Ok(guest_dtb) -} - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -fn get_developer_provided_dtb( - vm_cfg: &AxVMConfig, - crate_config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> AxResult>> { - match crate_config.kernel.image_location.as_deref() { - Some("memory") => { - let vm_images = provider - .static_vm_images() - .iter() - .find(|&v| v.id == vm_cfg.id()); - - if let Some(dtb) = vm_images.and_then(|images| images.dtb) { - info!("DTB file in memory, size: 0x{:x}", dtb.len()); - return Ok(Some(dtb.to_vec())); - } - } - #[cfg(any(feature = "fs", feature = "host-fs"))] - Some("fs") => { - if let Some(dtb_path) = &crate_config.kernel.dtb_path { - let dtb_buffer = crate::boot::images::fs::read_full_image(dtb_path, provider)?; - info!("DTB file in fs, size: 0x{:x}", dtb_buffer.len()); - return Ok(Some(dtb_buffer)); - } - } - _ => { - return ax_errno::ax_err!( - InvalidInput, - "Unsupported image_location; use \"memory\" or enable fs feature for \"fs\"" - ); - } - } - Ok(None) -} diff --git a/virtualization/axvm/src/arch/aarch64/capabilities.rs b/virtualization/axvm/src/arch/aarch64/capabilities.rs index 364defd376..ca370c3731 100644 --- a/virtualization/axvm/src/arch/aarch64/capabilities.rs +++ b/virtualization/axvm/src/arch/aarch64/capabilities.rs @@ -8,7 +8,7 @@ use axvm_types::VMInterruptMode; use super::Aarch64Arch; use crate::architecture::{ - AddressSpacePlatform, BootImagePlatform, DevicePlatform, HostTimePlatform, + AddressSpacePlatform, BootImagePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, }; impl DevicePlatform for Aarch64Arch { @@ -38,7 +38,17 @@ impl BootImagePlatform for Aarch64Arch { let bytes = dtb.as_bytes(); let source = core::ptr::NonNull::new(bytes.as_ptr() as *mut u8) .ok_or_else(|| ax_err_type!(InvalidData, "Guest DTB pointer is null"))?; - crate::boot::fdt::update_fdt(source, bytes.len(), loader.vm.clone(), &loader.config) + super::fdt::core::update_fdt(source, bytes.len(), loader.vm.clone(), &loader.config) + } +} + +impl GuestBootPlatform for Aarch64Arch { + fn prepare_guest_boot( + vm_config: &mut crate::config::AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn crate::boot::BootImageProvider, + ) -> AxResult> { + super::fdt::core::prepare_dtb_guest(vm_config, vm_create_config, provider) } } @@ -50,6 +60,52 @@ pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::Vir ax_std::os::arceos::modules::ax_hal::mem::phys_to_virt(paddr) } +pub(super) fn decode_gic_spi(specifier: &[u32]) -> Option { + (specifier.first().copied() == Some(0)) + .then(|| specifier.get(1).copied()) + .flatten() +} + +pub(super) fn patch_runtime_fdt( + fdt_bytes: &[u8], + vm: &crate::AxVMRef, + crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + let initrd = vm.with_config(|config| { + super::fdt::initrd_start_size_from_image_config(config.image_config.ramdisk.as_ref()) + }); + super::fdt::core::patch_guest_fdt_for_runtime( + fdt_bytes, + &vm.memory_regions(), + crate_config, + initrd, + true, + ) +} + +pub(super) fn patch_provided_fdt( + provided_dtb: &[u8], + host_dtb: Option<&[u8]>, + crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + let provided_fdt = fdt_edit::Fdt::from_bytes(provided_dtb).map_err(|err| { + ax_err_type!( + InvalidData, + format!("Failed to parse provided DTB image: {err:#?}") + ) + })?; + let host_fdt = host_dtb + .map(fdt_edit::Fdt::from_bytes) + .transpose() + .map_err(|err| { + ax_err_type!( + InvalidData, + format!("Failed to parse host DTB image: {err:#?}") + ) + })?; + super::fdt::update_cpu_node(&provided_fdt, host_fdt.as_ref(), crate_config) +} + fn assign_passthrough_spis( vm: &crate::AxVM, config: &crate::config::AxVMConfig, diff --git a/virtualization/axvm/src/arch/aarch64/fdt.rs b/virtualization/axvm/src/arch/aarch64/fdt.rs new file mode 100644 index 0000000000..a3c0db1fb5 --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/fdt.rs @@ -0,0 +1,97 @@ +//! AArch64 compatibility facade and target-specific guest FDT policy. + +use alloc::vec::Vec; + +use ax_errno::{AxResult, ax_err_type}; +use fdt_edit::Fdt; + +use crate::{ + boot::{BootImageProvider, fdt::GuestDtbImage}, + config::AxVMConfig, +}; + +#[path = "../../boot/fdt/core/mod.rs"] +pub(crate) mod core; + +pub use core::{ + parse_passthrough_devices_address, parse_reserved_memory_regions, parse_vm_interrupt, + reserve_excluded_device_ranges, set_phys_cpu_sets, setup_guest_fdt_from_vmm, try_get_host_fdt, + update_fdt, update_provided_fdt, +}; + +pub(crate) fn guest_fdt_policy() -> core::GuestFdtPolicy { + core::GuestFdtPolicy { + patch_runtime: super::capabilities::patch_runtime_fdt, + patch_provided: super::capabilities::patch_provided_fdt, + decode_interrupt: super::capabilities::decode_gic_spi, + } +} + +pub(crate) fn host_fdt_bootarg() -> usize { + super::capabilities::host_fdt_bootarg() +} + +pub(crate) fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + super::capabilities::host_phys_to_virt(paddr) +} + +pub(super) fn initrd_start_size_from_image_config( + ramdisk: Option<&crate::config::RamdiskInfo>, +) -> Option<(u64, u64)> { + let ramdisk = ramdisk?; + Some((ramdisk.load_gpa.as_usize() as u64, ramdisk.size? as u64)) +} + +pub(super) fn update_cpu_node( + fdt: &Fdt, + host_fdt: Option<&Fdt>, + crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + let Some(host_fdt) = host_fdt else { + return Ok(fdt.encode().as_ref().to_vec()); + }; + + let phys_cpu_ids = crate_config + .base + .phys_cpu_ids + .as_deref() + .ok_or_else(|| ax_err_type!(InvalidInput, "phys_cpu_ids is missing"))?; + let mut tree = core::tree::FdtTree::from_fdt(fdt.clone()); + tree.inner_mut().remove_by_path("/cpus"); + + if let Some(host_cpus_id) = host_fdt.get_by_path_id("/cpus") { + let cpus_id = + tree.copy_subtree_from(host_fdt, host_cpus_id, tree.inner().root_id(), true)?; + let cpu_paths = tree + .node_paths() + .into_iter() + .filter_map(|(id, path)| { + (path.starts_with("/cpus/cpu@") + && !core::create::need_cpu_node(phys_cpu_ids, tree.inner(), id, &path)) + .then_some(path) + }) + .collect::>(); + for path in cpu_paths { + tree.inner_mut().remove_by_path(&path); + } + if let Some(cpus) = tree.inner_mut().node_mut(cpus_id) { + for property in [ + "riscv,cbop-block-size", + "riscv,cboz-block-size", + "riscv,cbom-block-size", + ] { + cpus.remove_property(property); + } + } + } + + Ok(tree.finish()) +} + +pub fn handle_fdt_operations( + vm_config: &mut AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult> { + core::prepare_dtb_guest(vm_config, vm_create_config, provider) +} diff --git a/virtualization/axvm/src/arch/aarch64/mod.rs b/virtualization/axvm/src/arch/aarch64/mod.rs index 962c4f5d0e..5ca6894931 100644 --- a/virtualization/axvm/src/arch/aarch64/mod.rs +++ b/virtualization/axvm/src/arch/aarch64/mod.rs @@ -32,6 +32,7 @@ use crate::{ mod capabilities; #[path = "../../architecture/cpu_up.rs"] mod cpu_up; +pub(crate) mod fdt; mod gic; mod images; mod ipi; @@ -499,7 +500,7 @@ impl ArmVgicHostIf for ArmVgicHostIfImpl { } fn register_timer(deadline: Duration, callback: Box) { - let _ = default_host().register_timer(deadline.as_nanos() as u64, callback); + let _ = crate::timer::register_timer(deadline.as_nanos() as u64, callback); } fn read_vgicd_iidr() -> u32 { diff --git a/virtualization/axvm/src/arch/loongarch64/capabilities.rs b/virtualization/axvm/src/arch/loongarch64/capabilities.rs index c712d6d8f1..14472f98a3 100644 --- a/virtualization/axvm/src/arch/loongarch64/capabilities.rs +++ b/virtualization/axvm/src/arch/loongarch64/capabilities.rs @@ -1,12 +1,35 @@ //! LoongArch64 implementations of AxVM platform capability hooks. use super::LoongArch64Arch; -use crate::architecture::{AddressSpacePlatform, DevicePlatform, HostTimePlatform}; +use crate::architecture::{ + AddressSpacePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, +}; impl DevicePlatform for LoongArch64Arch {} impl AddressSpacePlatform for LoongArch64Arch {} +impl GuestBootPlatform for LoongArch64Arch { + fn init_guest_boot_resources() { + super::boot::init(); + } + + fn prepare_guest_boot( + vm_config: &mut crate::config::AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + _provider: &dyn crate::boot::BootImageProvider, + ) -> ax_errno::AxResult> { + if vm_create_config.kernel.effective_boot_protocol() != axvmconfig::VMBootProtocol::Uefi { + return ax_errno::ax_err!( + Unsupported, + "LoongArch AxVisor guests currently require UEFI boot" + ); + } + super::boot::prepare_uefi_fdt_config(vm_config, vm_create_config)?; + Ok(None) + } +} + impl HostTimePlatform for LoongArch64Arch { fn set_oneshot_timer(_deadline_ns: u64) {} diff --git a/virtualization/axvm/src/arch/loongarch64/fdt.rs b/virtualization/axvm/src/arch/loongarch64/fdt.rs new file mode 100644 index 0000000000..eca32a8fd2 --- /dev/null +++ b/virtualization/axvm/src/arch/loongarch64/fdt.rs @@ -0,0 +1,24 @@ +//! LoongArch compatibility facade for UEFI guest FDT preparation. + +use ax_errno::AxResult; +use axvmconfig::{AxVMCrateConfig, VMBootProtocol}; + +use crate::config::AxVMConfig; + +pub fn init_guest_boot_resources() { + super::boot::init(); +} + +pub fn handle_fdt_operations( + vm_config: &mut AxVMConfig, + vm_create_config: &mut AxVMCrateConfig, +) -> AxResult { + if vm_create_config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { + return super::boot::prepare_uefi_fdt_config(vm_config, vm_create_config); + } + + ax_errno::ax_err!( + Unsupported, + "LoongArch AxVisor guests currently require UEFI boot" + ) +} diff --git a/virtualization/axvm/src/arch/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/mod.rs index 87aa9d4d1f..fa0c8bcb89 100644 --- a/virtualization/axvm/src/arch/loongarch64/mod.rs +++ b/virtualization/axvm/src/arch/loongarch64/mod.rs @@ -23,6 +23,7 @@ use crate::host::{HostMemory, HostTime, default_host}; pub(crate) mod boot; mod capabilities; +pub(crate) mod fdt; mod idle; pub(crate) mod irq; mod npt; @@ -344,7 +345,7 @@ impl LoongArchHostOps for AxvmLoongArchHostOps { deadline: Duration, callback: Box, ) -> usize { - default_host().register_timer(deadline.as_nanos() as u64, callback) + crate::timer::register_timer(deadline.as_nanos() as u64, callback) } fn cancel_timer(token: usize) { diff --git a/virtualization/axvm/src/arch/mod.rs b/virtualization/axvm/src/arch/mod.rs index fdab0d56b5..cf445c9b5f 100644 --- a/virtualization/axvm/src/arch/mod.rs +++ b/virtualization/axvm/src/arch/mod.rs @@ -6,7 +6,7 @@ use axvm_types::NestedPagingConfig; pub(crate) use crate::architecture::*; use crate::architecture::{ - AddressSpacePlatform, BootImagePlatform, DevicePlatform, HostTimePlatform, + AddressSpacePlatform, BootImagePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, }; #[cfg(target_arch = "aarch64")] @@ -22,12 +22,16 @@ mod x86_64; pub(crate) use aarch64::Aarch64Arch as CurrentArch; #[cfg(target_arch = "aarch64")] pub use aarch64::ImageLoader; +#[cfg(target_arch = "aarch64")] +pub(crate) use aarch64::fdt; #[cfg(target_arch = "loongarch64")] pub(crate) use loongarch64::LoongArch64Arch as CurrentArch; #[cfg(target_arch = "loongarch64")] pub(crate) use loongarch64::boot as guest_platform; #[cfg(target_arch = "loongarch64")] pub use loongarch64::boot::ImageLoader; +#[cfg(target_arch = "loongarch64")] +pub(crate) use loongarch64::fdt; #[cfg(not(target_arch = "loongarch64"))] pub(crate) mod guest_platform { #[doc(hidden)] @@ -37,10 +41,14 @@ pub(crate) mod guest_platform { pub use riscv64::ImageLoader; #[cfg(target_arch = "riscv64")] pub(crate) use riscv64::Riscv64Arch as CurrentArch; +#[cfg(target_arch = "riscv64")] +pub(crate) use riscv64::fdt; #[cfg(target_arch = "x86_64")] pub(crate) use x86_64::X86_64Arch as CurrentArch; #[cfg(target_arch = "x86_64")] pub use x86_64::boot::ImageLoader; +#[cfg(target_arch = "x86_64")] +pub(crate) use x86_64::fdt; /// Architecture-specific public compatibility exports. pub mod platform { @@ -126,6 +134,18 @@ pub(crate) fn set_oneshot_timer(deadline_ns: u64) { CurrentArch::set_oneshot_timer(deadline_ns); } +pub(crate) fn init_guest_boot_resources() { + CurrentArch::init_guest_boot_resources(); +} + +pub(crate) fn prepare_guest_boot( + vm_config: &mut crate::config::AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn crate::boot::BootImageProvider, +) -> AxResult> { + CurrentArch::prepare_guest_boot(vm_config, vm_create_config, provider) +} + pub(crate) fn load_images_from_memory( loader: &mut crate::boot::images::ImageLoaderCore<'_>, images: crate::boot::StaticVmImage, @@ -146,3 +166,9 @@ pub(crate) fn is_x86_linux_image_config( ) -> bool { CurrentArch::is_x86_linux_image_config(config, provider) } + +pub(crate) fn default_boot_firmware_load_gpa( + config: &axvmconfig::AxVMCrateConfig, +) -> Option { + CurrentArch::default_boot_firmware_load_gpa(config) +} diff --git a/virtualization/axvm/src/arch/riscv64/capabilities.rs b/virtualization/axvm/src/arch/riscv64/capabilities.rs index 0fc0d173df..9d26752f75 100644 --- a/virtualization/axvm/src/arch/riscv64/capabilities.rs +++ b/virtualization/axvm/src/arch/riscv64/capabilities.rs @@ -1,10 +1,12 @@ //! RISC-V implementations of AxVM platform capability hooks. +use alloc::{format, vec::Vec}; + use ax_errno::{AxResult, ax_err_type}; use super::Riscv64Arch; use crate::architecture::{ - AddressSpacePlatform, BootImagePlatform, DevicePlatform, HostTimePlatform, + AddressSpacePlatform, BootImagePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, }; impl DevicePlatform for Riscv64Arch { @@ -29,7 +31,17 @@ impl BootImagePlatform for Riscv64Arch { let bytes = dtb.as_bytes(); let source = core::ptr::NonNull::new(bytes.as_ptr() as *mut u8) .ok_or_else(|| ax_err_type!(InvalidData, "Guest DTB pointer is null"))?; - crate::boot::fdt::update_fdt(source, bytes.len(), loader.vm.clone(), &loader.config) + super::fdt::core::update_fdt(source, bytes.len(), loader.vm.clone(), &loader.config) + } +} + +impl GuestBootPlatform for Riscv64Arch { + fn prepare_guest_boot( + vm_config: &mut crate::config::AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn crate::boot::BootImageProvider, + ) -> AxResult> { + super::fdt::core::prepare_dtb_guest(vm_config, vm_create_config, provider) } } @@ -40,3 +52,49 @@ pub fn host_fdt_bootarg() -> usize { pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { ax_std::os::arceos::modules::ax_hal::mem::phys_to_virt(paddr) } + +pub(super) fn decode_plic_source(specifier: &[u32]) -> Option { + specifier.first().copied().filter(|source| *source != 0) +} + +pub(super) fn patch_runtime_fdt( + fdt_bytes: &[u8], + vm: &crate::AxVMRef, + crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + let host_fdt = super::fdt::core::try_get_host_fdt() + .map(fdt_edit::Fdt::from_bytes) + .transpose() + .map_err(|err| { + ax_err_type!( + InvalidData, + format!("Failed to parse host FDT while updating guest FDT: {err:#?}") + ) + })?; + let guest_fdt = super::fdt::core::patch_guest_fdt_for_runtime( + fdt_bytes, + &vm.memory_regions(), + crate_config, + None, + false, + )?; + super::fdt::ensure_chosen_from_host(guest_fdt, host_fdt.as_ref()) +} + +pub(super) fn patch_provided_fdt( + provided_dtb: &[u8], + _host_dtb: Option<&[u8]>, + _crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + Ok(provided_dtb.to_vec()) +} + +#[cfg(test)] +mod tests { + #[test] + fn plic_interrupt_uses_first_nonzero_fdt_cell() { + assert_eq!(super::decode_plic_source(&[8]), Some(8)); + assert_eq!(super::decode_plic_source(&[0]), None); + assert_eq!(super::decode_plic_source(&[]), None); + } +} diff --git a/virtualization/axvm/src/arch/riscv64/fdt.rs b/virtualization/axvm/src/arch/riscv64/fdt.rs new file mode 100644 index 0000000000..8aa7a815b9 --- /dev/null +++ b/virtualization/axvm/src/arch/riscv64/fdt.rs @@ -0,0 +1,61 @@ +//! RISC-V compatibility facade and target-specific guest FDT policy. + +use alloc::vec::Vec; + +use ax_errno::AxResult; + +use crate::{ + boot::{BootImageProvider, fdt::GuestDtbImage}, + config::AxVMConfig, +}; + +#[path = "../../boot/fdt/core/mod.rs"] +pub(crate) mod core; + +pub use core::{ + parse_passthrough_devices_address, parse_reserved_memory_regions, parse_vm_interrupt, + reserve_excluded_device_ranges, set_phys_cpu_sets, setup_guest_fdt_from_vmm, try_get_host_fdt, + update_fdt, update_provided_fdt, +}; + +pub(crate) fn guest_fdt_policy() -> core::GuestFdtPolicy { + core::GuestFdtPolicy { + patch_runtime: super::capabilities::patch_runtime_fdt, + patch_provided: super::capabilities::patch_provided_fdt, + decode_interrupt: super::capabilities::decode_plic_source, + } +} + +pub(crate) fn host_fdt_bootarg() -> usize { + super::capabilities::host_fdt_bootarg() +} + +pub(crate) fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + super::capabilities::host_phys_to_virt(paddr) +} + +pub(super) fn ensure_chosen_from_host( + guest_dtb: Vec, + host_fdt: Option<&fdt_edit::Fdt>, +) -> AxResult> { + let Some(host_fdt) = host_fdt else { + return Ok(guest_dtb); + }; + let mut guest = core::tree::FdtTree::from_bytes(&guest_dtb)?; + if guest.inner().get_by_path_id("/chosen").is_some() { + return Ok(guest.finish()); + } + let Some(host_chosen) = host_fdt.get_by_path_id("/chosen") else { + return Ok(guest.finish()); + }; + guest.copy_subtree_from(host_fdt, host_chosen, guest.inner().root_id(), false)?; + Ok(guest.finish()) +} + +pub fn handle_fdt_operations( + vm_config: &mut AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult> { + core::prepare_dtb_guest(vm_config, vm_create_config, provider) +} diff --git a/virtualization/axvm/src/arch/riscv64/mod.rs b/virtualization/axvm/src/arch/riscv64/mod.rs index aa0e73bde0..eff1163d55 100644 --- a/virtualization/axvm/src/arch/riscv64/mod.rs +++ b/virtualization/axvm/src/arch/riscv64/mod.rs @@ -27,6 +27,7 @@ use crate::{ mod capabilities; #[path = "../../architecture/cpu_up.rs"] mod cpu_up; +pub(crate) mod fdt; mod images; mod irq; mod npt; diff --git a/virtualization/axvm/src/arch/x86_64/boot/mod.rs b/virtualization/axvm/src/arch/x86_64/boot/mod.rs index cd6bc5d8b2..45d6933ac5 100644 --- a/virtualization/axvm/src/arch/x86_64/boot/mod.rs +++ b/virtualization/axvm/src/arch/x86_64/boot/mod.rs @@ -45,6 +45,16 @@ impl<'a> ImageLoader<'a> { } impl BootImagePlatform for X86_64Arch { + fn default_boot_firmware_load_gpa( + config: &axvmconfig::AxVMCrateConfig, + ) -> Option { + const BUILT_IN_BIOS_LOAD_GPA: usize = 0x8000; + + (config.kernel.boot_firmware_path().is_none() + && config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot) + .then_some(GuestPhysAddr::from(BUILT_IN_BIOS_LOAD_GPA)) + } + fn load_images_from_memory( loader: &mut ImageLoaderCore<'_>, images: StaticVmImage, diff --git a/virtualization/axvm/src/arch/x86_64/capabilities.rs b/virtualization/axvm/src/arch/x86_64/capabilities.rs index 156df7c188..3584487dff 100644 --- a/virtualization/axvm/src/arch/x86_64/capabilities.rs +++ b/virtualization/axvm/src/arch/x86_64/capabilities.rs @@ -6,7 +6,9 @@ use ax_errno::{AxResult, ax_err_type}; use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; use super::X86_64Arch; -use crate::architecture::{AddressSpacePlatform, DevicePlatform, HostTimePlatform}; +use crate::architecture::{ + AddressSpacePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, +}; impl DevicePlatform for X86_64Arch { fn register_devices( @@ -62,3 +64,5 @@ impl AddressSpacePlatform for X86_64Arch { } impl HostTimePlatform for X86_64Arch {} + +impl GuestBootPlatform for X86_64Arch {} diff --git a/virtualization/axvm/src/arch/x86_64/fdt.rs b/virtualization/axvm/src/arch/x86_64/fdt.rs new file mode 100644 index 0000000000..7011d0f0c9 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/fdt.rs @@ -0,0 +1,4 @@ +//! x86_64 compatibility marker for the target-selected FDT facade. + +#[doc(hidden)] +pub const SUPPORTED: bool = false; diff --git a/virtualization/axvm/src/arch/x86_64/mod.rs b/virtualization/axvm/src/arch/x86_64/mod.rs index d35ae743f4..e519831cf6 100644 --- a/virtualization/axvm/src/arch/x86_64/mod.rs +++ b/virtualization/axvm/src/arch/x86_64/mod.rs @@ -28,7 +28,7 @@ use super::{ }; use crate::{ StopReason, - host::{HostMemory, HostTime, default_host}, + host::{HostMemory, default_host}, manager, vcpu::get_current_vcpu, }; @@ -36,6 +36,7 @@ use crate::{ pub(crate) mod boot; mod capabilities; mod exit; +pub(crate) mod fdt; mod host_irq; pub(crate) mod irq; mod npt; @@ -299,7 +300,7 @@ impl X86VlapicHostOps for AxvmX86HostOps { } fn register_timer(deadline_nanos: u64, callback: X86TimerCallback) -> Option { - Some(default_host().register_timer( + Some(crate::timer::register_timer( deadline_nanos, Box::new(move |deadline: Duration| callback(deadline.as_nanos() as u64)), )) diff --git a/virtualization/axvm/src/architecture/capabilities.rs b/virtualization/axvm/src/architecture/capabilities.rs index 52d407971a..e08ac927e7 100644 --- a/virtualization/axvm/src/architecture/capabilities.rs +++ b/virtualization/axvm/src/architecture/capabilities.rs @@ -4,8 +4,27 @@ use ax_errno::AxResult; use super::ArchOps; +/// Guest firmware preparation performed before common VM memory loading. +pub(crate) trait GuestBootPlatform { + fn init_guest_boot_resources() {} + + fn prepare_guest_boot( + _vm_config: &mut crate::config::AxVMConfig, + _vm_create_config: &mut axvmconfig::AxVMCrateConfig, + _provider: &dyn crate::boot::BootImageProvider, + ) -> AxResult> { + Ok(None) + } +} + /// Architecture-specific guest image planning layered over common byte loading. pub(crate) trait BootImagePlatform { + fn default_boot_firmware_load_gpa( + _config: &axvmconfig::AxVMCrateConfig, + ) -> Option { + None + } + fn load_images_from_memory( loader: &mut crate::boot::images::ImageLoaderCore<'_>, images: crate::boot::StaticVmImage, diff --git a/virtualization/axvm/src/architecture/mod.rs b/virtualization/axvm/src/architecture/mod.rs index 891b3601d4..8604f7770d 100644 --- a/virtualization/axvm/src/architecture/mod.rs +++ b/virtualization/axvm/src/architecture/mod.rs @@ -6,7 +6,7 @@ pub(crate) mod ops; mod types; pub(crate) use capabilities::{ - AddressSpacePlatform, BootImagePlatform, DevicePlatform, HostTimePlatform, + AddressSpacePlatform, BootImagePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, }; pub(crate) use exit::{handle_hypercall, handle_mmio_read, handle_mmio_write}; pub(crate) use ops::ArchOps; diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs b/virtualization/axvm/src/boot/fdt/core/create.rs similarity index 71% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs rename to virtualization/axvm/src/boot/fdt/core/create.rs index b9254f82bd..d6bc933669 100644 --- a/virtualization/axvm/src/arch/aarch64/boot/fdt/create.rs +++ b/virtualization/axvm/src/boot/fdt/core/create.rs @@ -12,25 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(target_arch = "riscv64")] -use alloc::format; use alloc::{string::String, vec::Vec}; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] use core::ptr::NonNull; use ax_errno::{AxResult, ax_err_type}; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] use ax_memory_addr::MemoryAddr; use axvmconfig::AxVMCrateConfig; use fdt_edit::{Fdt, Node, NodeId}; -use super::tree::FdtTree; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -use super::tree::GuestMemorySpec; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -use crate::VMMemoryRegion; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use crate::{AxVMRef, GuestPhysAddr, boot::images::load_vm_image_from_memory}; +use super::tree::{FdtTree, GuestMemorySpec}; +use crate::{AxVMRef, GuestPhysAddr, VMMemoryRegion, boot::images::load_vm_image_from_memory}; pub fn create_guest_fdt( fdt: &Fdt, @@ -115,7 +106,12 @@ fn cpu_reg_address(fdt: &Fdt, node_id: NodeId) -> Option { .and_then(|node| node.regs().first().map(|reg| reg.address as usize)) } -fn need_cpu_node(phys_cpu_ids: &[usize], fdt: &Fdt, node_id: NodeId, node_path: &str) -> bool { +pub(crate) fn need_cpu_node( + phys_cpu_ids: &[usize], + fdt: &Fdt, + node_id: NodeId, + node_path: &str, +) -> bool { if !node_path.starts_with("/cpus/cpu@") { return true; } @@ -130,7 +126,6 @@ fn need_cpu_node(phys_cpu_ids: &[usize], fdt: &Fdt, node_id: NodeId, node_path: }) } -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] fn guest_memory_specs( new_memory: &[VMMemoryRegion], crate_config: &AxVMCrateConfig, @@ -167,71 +162,31 @@ fn guest_memory_specs( .collect() } -#[cfg(any(test, target_arch = "aarch64"))] -fn initrd_start_size_from_image_config( - ramdisk: Option<&crate::config::RamdiskInfo>, -) -> Option<(u64, u64)> { - let rd = ramdisk?; - let start = rd.load_gpa.as_usize() as u64; - let size = rd.size? as u64; - Some((start, size)) -} - #[cfg(test)] fn initrd_range_from_image_config( ramdisk: Option<&crate::config::RamdiskInfo>, ) -> Option<(u64, u64)> { - let (start, size) = initrd_start_size_from_image_config(ramdisk)?; + let ramdisk = ramdisk?; + let start = ramdisk.load_gpa.as_usize() as u64; + let size = ramdisk.size? as u64; Some((start, start.saturating_add(size))) } -#[cfg(target_arch = "aarch64")] pub fn update_fdt( fdt_src: NonNull, dtb_size: usize, vm: AxVMRef, crate_config: &AxVMCrateConfig, ) -> AxResult { + let patch_runtime = super::selected_guest_fdt_policy().patch_runtime; + // SAFETY: `fdt_src` originates from `GuestDtbImage::as_bytes`, and the + // caller supplies the exact slice length while the image remains borrowed. let fdt_bytes = unsafe { core::slice::from_raw_parts(fdt_src.as_ptr(), dtb_size) }; - let initrd_start_size = vm.with_config(|config| { - initrd_start_size_from_image_config(config.image_config.ramdisk.as_ref()) - }); - let new_fdt_bytes = patch_guest_fdt_for_runtime( - fdt_bytes, - &vm.memory_regions(), - crate_config, - initrd_start_size, - true, - )?; + let new_fdt_bytes = patch_runtime(fdt_bytes, &vm, crate_config)?; load_patched_fdt(vm, new_fdt_bytes) } -#[cfg(target_arch = "riscv64")] -pub fn update_fdt( - fdt_src: NonNull, - dtb_size: usize, - vm: AxVMRef, - crate_config: &AxVMCrateConfig, -) -> AxResult { - let fdt_bytes = unsafe { core::slice::from_raw_parts(fdt_src.as_ptr(), dtb_size) }; - let host_fdt = super::try_get_host_fdt() - .map(Fdt::from_bytes) - .transpose() - .map_err(|e| { - ax_err_type!( - InvalidData, - format!("Failed to parse host FDT while updating guest FDT: {e:#?}") - ) - })?; - let new_fdt_bytes = - patch_guest_fdt_for_runtime(fdt_bytes, &vm.memory_regions(), crate_config, None, false)?; - let new_fdt_bytes = ensure_chosen_from_host(new_fdt_bytes, host_fdt.as_ref())?; - - load_patched_fdt(vm, new_fdt_bytes) -} - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] fn load_patched_fdt(vm: AxVMRef, new_fdt_bytes: Vec) -> AxResult { let dest_addr = calculate_dtb_load_addr(vm.clone(), new_fdt_bytes.len())?; debug!( @@ -243,8 +198,7 @@ fn load_patched_fdt(vm: AxVMRef, new_fdt_bytes: Vec) -> AxResult { vm.set_guest_device_tree(dest_addr, new_fdt_bytes) } -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -pub(crate) fn patch_guest_fdt_for_runtime( +pub fn patch_guest_fdt_for_runtime( fdt_bytes: &[u8], memory_regions: &[VMMemoryRegion], crate_config: &AxVMCrateConfig, @@ -263,23 +217,6 @@ pub(crate) fn patch_guest_fdt_for_runtime( Ok(tree.finish()) } -#[cfg(target_arch = "riscv64")] -fn ensure_chosen_from_host(guest_dtb: Vec, host_fdt: Option<&Fdt>) -> AxResult> { - let Some(host_fdt) = host_fdt else { - return Ok(guest_dtb); - }; - let mut guest = FdtTree::from_bytes(&guest_dtb)?; - if guest.inner().get_by_path_id("/chosen").is_some() { - return Ok(guest.finish()); - } - let Some(host_chosen) = host_fdt.get_by_path_id("/chosen") else { - return Ok(guest.finish()); - }; - guest.copy_subtree_from(host_fdt, host_chosen, guest.inner().root_id(), false)?; - Ok(guest.finish()) -} - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] pub(crate) fn calculate_dtb_load_addr(vm: AxVMRef, fdt_size: usize) -> AxResult { const MB: usize = 1024 * 1024; @@ -292,8 +229,12 @@ pub(crate) fn calculate_dtb_load_addr(vm: AxVMRef, fdt_size: usize) -> AxResult< let use_configured_dtb_addr = config.image_config.dtb_load_gpa.is_some() && !main_memory.is_identical(); - let dtb_addr = if use_configured_dtb_addr { - config.image_config.dtb_load_gpa.unwrap() + let dtb_addr = if let Some(configured) = config + .image_config + .dtb_load_gpa + .filter(|_| use_configured_dtb_addr) + { + configured } else { let main_memory_size = main_memory.size().min(512 * MB); let addr = (main_memory.gpa + main_memory_size - fdt_size).align_down(2 * MB); @@ -309,53 +250,6 @@ pub(crate) fn calculate_dtb_load_addr(vm: AxVMRef, fdt_size: usize) -> AxResult< Ok(dtb_addr) } -#[cfg(target_arch = "aarch64")] -pub fn update_cpu_node( - fdt: &Fdt, - host_fdt: Option<&Fdt>, - crate_config: &AxVMCrateConfig, -) -> AxResult> { - let Some(host_fdt) = host_fdt else { - return Ok(fdt.encode().as_ref().to_vec()); - }; - - let phys_cpu_ids = crate_config - .base - .phys_cpu_ids - .as_deref() - .ok_or_else(|| ax_err_type!(InvalidInput, "phys_cpu_ids is missing"))?; - let mut tree = FdtTree::from_fdt(fdt.clone()); - tree.inner_mut().remove_by_path("/cpus"); - - if let Some(host_cpus_id) = host_fdt.get_by_path_id("/cpus") { - let cpus_id = - tree.copy_subtree_from(host_fdt, host_cpus_id, tree.inner().root_id(), true)?; - let cpu_paths = tree - .node_paths() - .into_iter() - .filter_map(|(id, path)| { - (path.starts_with("/cpus/cpu@") - && !need_cpu_node(phys_cpu_ids, tree.inner(), id, &path)) - .then_some(path) - }) - .collect::>(); - for path in cpu_paths { - tree.inner_mut().remove_by_path(&path); - } - if let Some(cpus) = tree.inner_mut().node_mut(cpus_id) { - for prop in [ - "riscv,cbop-block-size", - "riscv,cboz-block-size", - "riscv,cbom-block-size", - ] { - cpus.remove_property(prop); - } - } - } - - Ok(tree.finish()) -} - #[cfg(test)] mod tests { use axvmconfig::AxVMCrateConfig; diff --git a/virtualization/axvm/src/boot/fdt/device.rs b/virtualization/axvm/src/boot/fdt/core/device.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/device.rs rename to virtualization/axvm/src/boot/fdt/core/device.rs diff --git a/virtualization/axvm/src/boot/fdt/core/mod.rs b/virtualization/axvm/src/boot/fdt/core/mod.rs new file mode 100644 index 0000000000..5f6c8736c9 --- /dev/null +++ b/virtualization/axvm/src/boot/fdt/core/mod.rs @@ -0,0 +1,174 @@ +//! Architecture-neutral guest device-tree preparation. + +use alloc::{format, vec::Vec}; + +use ax_errno::{AxResult, ax_err_type}; +use axvmconfig::{AxVMCrateConfig, VMBootProtocol}; + +use crate::{ + boot::{BootImageProvider, fdt::GuestDtbImage}, + config::AxVMConfig, +}; + +pub(crate) mod create; +mod device; +mod parser; +mod policy; +mod print; +pub(crate) mod tree; + +#[cfg(test)] +mod tree_tests; + +pub use create::{patch_guest_fdt_for_runtime, update_fdt}; +pub use parser::*; +pub use policy::GuestFdtPolicy; + +pub fn prepare_dtb_guest( + vm_config: &mut AxVMConfig, + vm_create_config: &mut AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult> { + if vm_create_config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { + skip_guest_dtb(vm_config, vm_create_config); + return Ok(None); + } + + let host_fdt_bytes = try_get_host_fdt(); + let guest_dtb = build_guest_dtb(vm_config, vm_create_config, provider, host_fdt_bytes)?; + enrich_guest_config(vm_config, vm_create_config, guest_dtb.as_ref())?; + Ok(guest_dtb) +} + +pub(crate) fn selected_guest_fdt_policy() -> GuestFdtPolicy { + super::guest_fdt_policy() +} + +fn skip_guest_dtb(vm_config: &mut AxVMConfig, vm_create_config: &mut AxVMCrateConfig) { + info!( + "VM[{}] uses UEFI boot protocol, skipping guest DTB handling", + vm_config.id() + ); + vm_config.clear_dtb_load_gpa(); + vm_create_config.kernel.dtb_load_addr = None; +} + +fn build_guest_dtb( + vm_config: &mut AxVMConfig, + vm_create_config: &mut AxVMCrateConfig, + provider: &dyn BootImageProvider, + host_fdt_bytes: Option<&'static [u8]>, +) -> AxResult> { + let provided_dtb = get_developer_provided_dtb(vm_config, vm_create_config, provider)?; + + match (host_fdt_bytes, provided_dtb) { + (Some(host_bytes), Some(provided)) => { + let host_fdt = parse_host_fdt(host_bytes)?; + set_phys_cpu_sets(vm_config, &host_fdt, vm_create_config)?; + info!("VM[{}] found DTB, parsing...", vm_config.id()); + reserve_excluded_device_ranges(vm_config, vm_create_config, &provided)?; + update_provided_fdt(&provided, Some(host_bytes), vm_create_config) + .map(GuestDtbImage::new) + .map(Some) + } + (Some(host_bytes), None) => { + let host_fdt = parse_host_fdt(host_bytes)?; + set_phys_cpu_sets(vm_config, &host_fdt, vm_create_config)?; + info!( + "VM[{}] DTB not found, generating from the VM configuration", + vm_config.id() + ); + setup_guest_fdt_from_vmm(host_bytes, vm_config, vm_create_config) + .map(GuestDtbImage::new) + .map(Some) + } + (None, Some(provided)) => { + info!("VM[{}] found DTB, parsing...", vm_config.id()); + reserve_excluded_device_ranges(vm_config, vm_create_config, &provided)?; + update_provided_fdt(&provided, None, vm_create_config) + .map(GuestDtbImage::new) + .map(Some) + } + (None, None) => { + warn!( + "VM[{}] no guest DTB provided; continuing without generated DTB", + vm_config.id() + ); + Ok(None) + } + } +} + +fn parse_host_fdt(host_fdt_bytes: &'static [u8]) -> AxResult { + fdt_edit::Fdt::from_bytes(host_fdt_bytes) + .map_err(|err| ax_err_type!(InvalidData, format!("Failed to parse host FDT: {err:#?}"))) +} + +fn enrich_guest_config( + vm_config: &mut AxVMConfig, + vm_create_config: &mut AxVMCrateConfig, + guest_dtb: Option<&GuestDtbImage>, +) -> AxResult { + let Some(dtb) = guest_dtb.map(GuestDtbImage::as_bytes) else { + clear_unresolved_dtb_config(vm_config, vm_create_config); + return Ok(()); + }; + + parse_reserved_memory_regions(vm_create_config, dtb)?; + parse_passthrough_devices_address(vm_config, vm_create_config, dtb)?; + parse_vm_interrupt(vm_config, dtb) +} + +fn clear_unresolved_dtb_config(vm_config: &mut AxVMConfig, vm_create_config: &mut AxVMCrateConfig) { + error!( + "VM[{}] DTB not found in memory, skipping...", + vm_config.id() + ); + let unresolved_devices = vm_config + .pass_through_devices() + .iter() + .filter(|device| device.length == 0) + .cloned() + .collect::>(); + if !unresolved_devices.is_empty() { + warn!( + "VM[{}] clearing {} unresolved passthrough discovery device(s)", + vm_config.id(), + unresolved_devices.len() + ); + for device in unresolved_devices { + vm_config.remove_pass_through_device(device); + } + } + vm_config.clear_dtb_load_gpa(); + vm_create_config.kernel.dtb_load_addr = None; +} + +fn get_developer_provided_dtb( + vm_config: &AxVMConfig, + crate_config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult>> { + match crate_config.kernel.image_location.as_deref() { + Some("memory") => Ok(provider + .static_vm_images() + .iter() + .find(|image| image.id == vm_config.id()) + .and_then(|images| images.dtb) + .map(|dtb| { + info!("DTB file in memory, size: 0x{:x}", dtb.len()); + dtb.to_vec() + })), + #[cfg(any(feature = "fs", feature = "host-fs"))] + Some("fs") => crate_config + .kernel + .dtb_path + .as_deref() + .map(|path| crate::boot::images::fs::read_full_image(path, provider)) + .transpose(), + _ => ax_errno::ax_err!( + InvalidInput, + "Unsupported image_location; use \"memory\" or enable fs feature for \"fs\"" + ), + } +} diff --git a/virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs b/virtualization/axvm/src/boot/fdt/core/parser.rs similarity index 90% rename from virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs rename to virtualization/axvm/src/boot/fdt/core/parser.rs index 1c7677c502..e7dda1b467 100644 --- a/virtualization/axvm/src/arch/aarch64/boot/fdt/parser.rs +++ b/virtualization/axvm/src/boot/fdt/core/parser.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! FDT parsing and processing functionality. +//! Architecture-neutral FDT parsing and guest configuration enrichment. use alloc::{ format, @@ -27,21 +27,18 @@ use axvmconfig::{ }; use fdt_edit::{Fdt, Node, NodeType, PciRange, PciSpace}; -#[cfg(target_arch = "aarch64")] -use super::create::update_cpu_node; use crate::{MappingFlags, config::AxVMConfig}; const PAGE_SIZE_4K: usize = 0x1000; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] pub fn try_get_host_fdt() -> Option<&'static [u8]> { - let bootarg: usize = crate::host_fdt_bootarg(); + let bootarg = super::super::host_fdt_bootarg(); if bootarg == 0 { warn!("Boot argument does not contain a host FDT pointer"); return None; } - let fdt_vaddr = crate::host_phys_to_virt(bootarg.into()); + let fdt_vaddr = super::super::host_phys_to_virt(bootarg.into()); super::tree::host_fdt_bytes_from_ptr(fdt_vaddr.as_ptr()).inspect(|bytes| { trace!("Host FDT size: 0x{:x}", bytes.len()); }) @@ -584,8 +581,8 @@ pub fn parse_passthrough_devices_address( Ok(()) } -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] pub fn parse_vm_interrupt(vm_cfg: &mut AxVMConfig, dtb: &[u8]) -> AxResult { + let decode_interrupt = super::selected_guest_fdt_policy().decode_interrupt; let fdt = Fdt::from_bytes(dtb).map_err(|e| { ax_err_type!( InvalidData, @@ -610,7 +607,7 @@ pub fn parse_vm_interrupt(vm_cfg: &mut AxVMConfig, dtb: &[u8]) -> AxResult { continue; }; for interrupt in view.interrupts() { - if let Some(irq) = passthrough_irq_from_interrupt_specifier(&interrupt.specifier) { + if let Some(irq) = decode_interrupt(&interrupt.specifier) { trace!("node: {name}, passthrough interrupt id: 0x{irq:x}"); vm_cfg.add_pass_through_irq(irq); } @@ -620,58 +617,13 @@ pub fn parse_vm_interrupt(vm_cfg: &mut AxVMConfig, dtb: &[u8]) -> AxResult { Ok(()) } -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -fn passthrough_irq_from_interrupt_specifier(specifier: &[u32]) -> Option { - #[cfg(target_arch = "aarch64")] - { - aarch64_gic_spi_from_interrupt_specifier(specifier) - } - #[cfg(target_arch = "riscv64")] - { - riscv_plic_source_from_interrupt_specifier(specifier) - } -} - -#[cfg(target_arch = "aarch64")] -fn aarch64_gic_spi_from_interrupt_specifier(specifier: &[u32]) -> Option { - (specifier.first().copied() == Some(0)) - .then(|| specifier.get(1).copied()) - .flatten() -} - -#[cfg(any(target_arch = "riscv64", test))] -fn riscv_plic_source_from_interrupt_specifier(specifier: &[u32]) -> Option { - specifier.first().copied().filter(|source| *source != 0) -} - -#[cfg(target_arch = "riscv64")] -pub fn update_provided_fdt( - provided_dtb: &[u8], - _host_dtb: Option<&[u8]>, - _crate_config: &AxVMCrateConfig, -) -> AxResult> { - Ok(provided_dtb.to_vec()) -} - -#[cfg(target_arch = "aarch64")] pub fn update_provided_fdt( provided_dtb: &[u8], host_dtb: Option<&[u8]>, crate_config: &AxVMCrateConfig, ) -> AxResult> { - let provided_fdt = Fdt::from_bytes(provided_dtb).map_err(|e| { - ax_err_type!( - InvalidData, - format!("Failed to parse provided DTB image: {e:#?}") - ) - })?; - let host_fdt = host_dtb.map(Fdt::from_bytes).transpose().map_err(|e| { - ax_err_type!( - InvalidData, - format!("Failed to parse host DTB image: {e:#?}") - ) - })?; - update_cpu_node(&provided_fdt, host_fdt.as_ref(), crate_config) + let patch_provided = super::selected_guest_fdt_policy().patch_provided; + patch_provided(provided_dtb, host_dtb, crate_config) } #[cfg(test)] @@ -803,21 +755,4 @@ mod tests { assert_eq!(ranges[0].base_gpa, 0x1000_1000); assert_eq!(ranges[0].length, 0x1000); } - - #[test] - fn riscv_plic_interrupt_uses_first_fdt_cell() { - assert_eq!( - super::riscv_plic_source_from_interrupt_specifier(&[8]), - Some(8) - ); - } - - #[test] - fn riscv_plic_interrupt_rejects_reserved_source_zero() { - assert_eq!( - super::riscv_plic_source_from_interrupt_specifier(&[0]), - None - ); - assert_eq!(super::riscv_plic_source_from_interrupt_specifier(&[]), None); - } } diff --git a/virtualization/axvm/src/boot/fdt/core/policy.rs b/virtualization/axvm/src/boot/fdt/core/policy.rs new file mode 100644 index 0000000000..2ed33515a1 --- /dev/null +++ b/virtualization/axvm/src/boot/fdt/core/policy.rs @@ -0,0 +1,17 @@ +//! Static target policies consumed by common guest FDT operations. + +use alloc::vec::Vec; + +use ax_errno::AxResult; +use axvmconfig::AxVMCrateConfig; + +pub type RuntimeFdtPatch = fn(&[u8], &crate::AxVMRef, &AxVMCrateConfig) -> AxResult>; +pub type ProvidedFdtPatch = fn(&[u8], Option<&[u8]>, &AxVMCrateConfig) -> AxResult>; + +/// Architecture operations required by common guest FDT processing. +#[derive(Clone, Copy)] +pub struct GuestFdtPolicy { + pub patch_runtime: RuntimeFdtPatch, + pub patch_provided: ProvidedFdtPatch, + pub decode_interrupt: fn(&[u32]) -> Option, +} diff --git a/virtualization/axvm/src/boot/fdt/print.rs b/virtualization/axvm/src/boot/fdt/core/print.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/print.rs rename to virtualization/axvm/src/boot/fdt/core/print.rs diff --git a/virtualization/axvm/src/boot/fdt/tree.rs b/virtualization/axvm/src/boot/fdt/core/tree.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/tree.rs rename to virtualization/axvm/src/boot/fdt/core/tree.rs diff --git a/virtualization/axvm/src/boot/fdt/tree_tests.rs b/virtualization/axvm/src/boot/fdt/core/tree_tests.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/tree_tests.rs rename to virtualization/axvm/src/boot/fdt/core/tree_tests.rs diff --git a/virtualization/axvm/src/boot/fdt/mod.rs b/virtualization/axvm/src/boot/fdt/mod.rs index 184157b52e..f3cc239b03 100644 --- a/virtualization/axvm/src/boot/fdt/mod.rs +++ b/virtualization/axvm/src/boot/fdt/mod.rs @@ -1,6 +1,64 @@ -//! Shared FDT source root and the selected architecture's guest boot facade. +//! Guest device-tree artifact and selected architecture compatibility facade. -#[path = "../../arch/aarch64/boot/fdt/mod.rs"] -mod platform; +use alloc::vec::Vec; -pub use platform::*; +pub use crate::arch::fdt::*; + +#[cfg(test)] +#[path = "core/mod.rs"] +pub mod test_core; + +/// Guest DTB artifact produced or patched before AxVM owns it. +#[derive(Debug, Clone)] +pub struct GuestDtbImage { + bytes: Vec, +} + +impl GuestDtbImage { + /// Wraps finalized guest DTB bytes. + pub fn new(bytes: Vec) -> Self { + Self { bytes } + } + + /// Returns the encoded guest DTB. + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } +} + +#[cfg(test)] +fn guest_fdt_policy() -> test_core::GuestFdtPolicy { + test_core::GuestFdtPolicy { + patch_runtime: test_runtime_patch, + patch_provided: test_provided_patch, + decode_interrupt: |specifier| specifier.first().copied(), + } +} + +#[cfg(test)] +fn host_fdt_bootarg() -> usize { + 0 +} + +#[cfg(test)] +fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + ax_memory_addr::VirtAddr::from(paddr.as_usize()) +} + +#[cfg(test)] +fn test_runtime_patch( + fdt: &[u8], + _vm: &crate::AxVMRef, + _config: &axvmconfig::AxVMCrateConfig, +) -> ax_errno::AxResult> { + Ok(fdt.to_vec()) +} + +#[cfg(test)] +fn test_provided_patch( + fdt: &[u8], + _host_fdt: Option<&[u8]>, + _config: &axvmconfig::AxVMCrateConfig, +) -> ax_errno::AxResult> { + Ok(fdt.to_vec()) +} diff --git a/virtualization/axvm/src/boot/mod.rs b/virtualization/axvm/src/boot/mod.rs index 44237fa56e..67379dc316 100644 --- a/virtualization/axvm/src/boot/mod.rs +++ b/virtualization/axvm/src/boot/mod.rs @@ -7,9 +7,19 @@ pub mod fdt; pub mod guest_platform; pub mod images; mod policy; +mod prepared; pub use images::*; -pub use policy::{GuestAcpiTables, GuestBootDescription, GuestDeviceTree, GuestFdtBuilder}; +pub use policy::{ + GuestAcpiTables, GuestBootDescription, GuestDeviceTree, GuestFdtBuilder, + boot_firmware_load_gpa, guest_boot_policy, +}; +pub use prepared::{PreparedGuestBoot, prepare_guest_boot}; + +/// Initializes architecture-owned guest firmware resources. +pub fn init_guest_boot_resources() { + crate::arch::init_guest_boot_resources(); +} /// Build-time image bytes supplied by the hypervisor application. #[derive(Clone, Copy, Debug)] diff --git a/virtualization/axvm/src/boot/policy.rs b/virtualization/axvm/src/boot/policy.rs index 14d429b3b5..305e410431 100644 --- a/virtualization/axvm/src/boot/policy.rs +++ b/virtualization/axvm/src/boot/policy.rs @@ -18,6 +18,35 @@ use alloc::vec::Vec; use axvm_types::GuestPhysAddr; +use super::BootImageProvider; + +/// Selects the guest address-adjustment policy for the current architecture. +pub fn guest_boot_policy( + config: &axvmconfig::AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> crate::config::GuestBootPolicy { + if crate::boot::is_x86_linux_image_config(config, provider) { + crate::config::GuestBootPolicy::KeepConfigured + } else { + crate::config::GuestBootPolicy::AdjustKernelForBootProtocol { + protocol: config.kernel.effective_boot_protocol(), + } + } +} + +/// Resolves the configured or architecture-default boot firmware load address. +pub fn boot_firmware_load_gpa(config: &axvmconfig::AxVMCrateConfig) -> Option { + if !config.kernel.enable_bios { + return None; + } + + config + .kernel + .bios_load_addr + .map(GuestPhysAddr::from) + .or_else(|| crate::arch::default_boot_firmware_load_gpa(config)) +} + /// Device-tree boot description owned by the VM lifecycle. #[derive(Debug, Clone)] pub struct GuestDeviceTree { diff --git a/virtualization/axvm/src/boot/prepared.rs b/virtualization/axvm/src/boot/prepared.rs new file mode 100644 index 0000000000..ca34bfd866 --- /dev/null +++ b/virtualization/axvm/src/boot/prepared.rs @@ -0,0 +1,53 @@ +//! Typed guest boot preparation shared by monitor integrations. + +use ax_errno::AxResult; +use axvmconfig::AxVMCrateConfig; + +use super::{BootImageProvider, fdt::GuestDtbImage, images::ImageLoaderCore}; +use crate::{AxVMRef, VMMemoryRegion, config::AxVMConfig}; + +/// Architecture-prepared VM configuration and optional guest DTB. +#[derive(Debug)] +pub struct PreparedGuestBoot { + config: AxVMCrateConfig, + guest_dtb: Option, +} + +impl PreparedGuestBoot { + /// Returns the architecture-enriched VM configuration. + pub const fn config(&self) -> &AxVMCrateConfig { + &self.config + } + + /// Loads all configured guest images into prepared VM memory. + /// + /// # Errors + /// + /// Returns an error when an image source is unavailable, an image layout is + /// invalid, or guest memory cannot hold the configured image. + pub fn load_images( + self, + main_memory: VMMemoryRegion, + vm: AxVMRef, + provider: &dyn BootImageProvider, + ) -> AxResult { + let mut loader = + ImageLoaderCore::new(main_memory, self.config, vm, provider, self.guest_dtb); + loader.load() + } +} + +/// Applies architecture boot preparation and returns a typed load request. +/// +/// # Errors +/// +/// Returns an error when firmware requirements are unsupported or guest boot +/// metadata cannot be parsed and validated. +pub fn prepare_guest_boot( + vm_config: &mut AxVMConfig, + mut config: AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult { + let guest_dtb = crate::arch::prepare_guest_boot(vm_config, &mut config, provider)?; + Ok(PreparedGuestBoot { config, guest_dtb }) +} diff --git a/virtualization/axvm/src/host/arceos.rs b/virtualization/axvm/src/host/arceos.rs index 8fd7464d5b..cb9069dd1e 100644 --- a/virtualization/axvm/src/host/arceos.rs +++ b/virtualization/axvm/src/host/arceos.rs @@ -2,7 +2,6 @@ extern crate alloc; -use alloc::boxed::Box; use core::{ sync::atomic::{AtomicUsize, Ordering}, time::Duration, @@ -81,8 +80,6 @@ impl HostMemory for ArceOsHost { } impl HostTime for ArceOsHost { - type CancelToken = usize; - fn monotonic_time(&self) -> Duration { modules::ax_hal::time::monotonic_time() } @@ -90,14 +87,6 @@ impl HostTime for ArceOsHost { fn set_oneshot_timer(&self, deadline_ns: u64) { crate::arch::set_oneshot_timer(deadline_ns); } - - fn register_timer( - &self, - deadline_ns: u64, - callback: Box, - ) -> Self::CancelToken { - crate::timer::register_timer(deadline_ns, callback) - } } pub(crate) fn dispatch_host_irq(vector: usize) { diff --git a/virtualization/axvm/src/host/traits.rs b/virtualization/axvm/src/host/traits.rs index 54c6582143..34e9f2f687 100644 --- a/virtualization/axvm/src/host/traits.rs +++ b/virtualization/axvm/src/host/traits.rs @@ -1,8 +1,5 @@ //! Internal host capability traits used by the AxVM runtime. -extern crate alloc; - -use alloc::boxed::Box; use core::time::Duration; use ax_errno::AxResult; @@ -35,21 +32,11 @@ pub trait HostMemory { /// Host time and timer operations. pub trait HostTime { - /// Timer cancellation token. - type CancelToken: Copy + Send + Sync + 'static; - /// Read monotonic host time. fn monotonic_time(&self) -> Duration; /// Program the host one-shot timer. fn set_oneshot_timer(&self, deadline_ns: u64); - - /// Register a VM timer callback. - fn register_timer( - &self, - deadline_ns: u64, - callback: Box, - ) -> Self::CancelToken; } /// Host CPU topology and affinity operations. diff --git a/virtualization/axvm/tests/arch_boundary_contract.rs b/virtualization/axvm/tests/arch_boundary_contract.rs index 04653d34aa..99ea20289e 100644 --- a/virtualization/axvm/tests/arch_boundary_contract.rs +++ b/virtualization/axvm/tests/arch_boundary_contract.rs @@ -116,6 +116,88 @@ fn common_domains_live_outside_architecture_directories() { } } +#[test] +fn common_modules_do_not_include_architecture_sources() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut violations = Vec::new(); + find_source_files(&source_root, &mut |path, source| { + if !path.starts_with(source_root.join("arch")) + && source.contains("#[path") + && source.contains("arch/") + { + violations.push(source_relative_path(&source_root, path)); + } + }); + + assert!( + violations.is_empty(), + "common modules must not include implementations from src/arch: {}", + violations.join(", ") + ); +} + +#[test] +fn architecture_directories_only_select_their_own_target() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let arch_root = source_root.join("arch"); + let architectures = ["aarch64", "loongarch64", "riscv64", "x86_64"]; + let mut violations = Vec::new(); + + for architecture in architectures { + find_source_files(&arch_root.join(architecture), &mut |path, source| { + for other_architecture in architectures { + if other_architecture != architecture + && source.contains(&format!("target_arch = \"{other_architecture}\"")) + { + violations.push(format!( + "{} selects {other_architecture}", + source_relative_path(&source_root, path) + )); + } + } + }); + } + + assert!( + violations.is_empty(), + "an architecture directory must not select another target: {}", + violations.join(", ") + ); +} + +#[test] +fn axvisor_vm_creation_uses_unified_guest_boot_facade() { + let axvisor_config = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../os/axvisor/src/config.rs"); + let source = std::fs::read_to_string(&axvisor_config) + .expect("Axvisor VM creation source must be readable"); + + for legacy_call in [ + "handle_fdt_operations", + "ImageLoader::new", + "x86_linux_direct_boot_config", + "DEFAULT_X86_BIOS_LOAD_GPA", + ] { + assert!( + !source.contains(legacy_call), + "Axvisor VM creation must use the unified AxVM boot facade: {legacy_call}" + ); + } +} + +#[test] +fn host_time_trait_only_exposes_common_clock_capabilities() { + let host_traits = include_str!("../src/host/traits.rs"); + + for architecture_specific_detail in ["CancelToken", "fn register_timer"] { + assert!( + !host_traits.contains(architecture_specific_detail), + "HostTime must not expose architecture-specific timer details: \ + {architecture_specific_detail}" + ); + } +} + fn find_target_arch_cfg_outside_arch( source_root: &std::path::Path, directory: &std::path::Path, @@ -146,3 +228,23 @@ fn find_target_arch_cfg_outside_arch( } violations } + +fn find_source_files(directory: &std::path::Path, visit: &mut impl FnMut(&std::path::Path, &str)) { + for entry in std::fs::read_dir(directory).expect("AxVM source directory must be readable") { + let entry = entry.expect("AxVM source directory entry must be readable"); + let path = entry.path(); + if path.is_dir() { + find_source_files(&path, visit); + } else if path.extension().is_some_and(|extension| extension == "rs") { + let source = std::fs::read_to_string(&path).expect("AxVM source file must be readable"); + visit(&path, &source); + } + } +} + +fn source_relative_path(source_root: &std::path::Path, path: &std::path::Path) -> String { + path.strip_prefix(source_root) + .expect("source path must be below src") + .display() + .to_string() +} From d07490992571cfe557f9d49e03d5ddf6c167c1de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Fri, 10 Jul 2026 17:53:04 +0800 Subject: [PATCH 4/5] refactor(axvm): tighten vm module boundaries --- virtualization/axvm/src/arch/aarch64/mod.rs | 9 +++--- .../axvm/src/arch/loongarch64/mod.rs | 11 +++---- virtualization/axvm/src/arch/riscv64/mod.rs | 6 ++-- virtualization/axvm/src/arch/x86_64/mod.rs | 25 +++++++-------- virtualization/axvm/src/architecture/mod.rs | 1 - virtualization/axvm/src/architecture/ops.rs | 5 +-- virtualization/axvm/src/architecture/types.rs | 30 +---------------- virtualization/axvm/src/config.rs | 21 ++++++++++++ virtualization/axvm/src/{vm.rs => vm/mod.rs} | 2 ++ virtualization/axvm/src/vm/prepare/vcpus.rs | 31 +++--------------- .../axvm/tests/arch_boundary_contract.rs | 32 +++++++++++++++++-- 11 files changed, 85 insertions(+), 88 deletions(-) rename virtualization/axvm/src/{vm.rs => vm/mod.rs} (99%) diff --git a/virtualization/axvm/src/arch/aarch64/mod.rs b/virtualization/axvm/src/arch/aarch64/mod.rs index 5ca6894931..5bbcedbb34 100644 --- a/virtualization/axvm/src/arch/aarch64/mod.rs +++ b/virtualization/axvm/src/arch/aarch64/mod.rs @@ -22,7 +22,7 @@ use axvm_types::{ use super::{ ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, VcpuSetupContext, + VcpuRunAction, }; use crate::{ architecture::ops::target_phys_cpu_ids, @@ -147,11 +147,10 @@ impl ArchOps for Aarch64Arch { } fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, + config: &crate::config::AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], ) -> AxResult<::SetupConfig> { - let (interrupt_mode, _console, _ports, _memory_regions, _firmware_boot) = - ctx.into_parts(); - let passthrough = interrupt_mode == axvm_types::VMInterruptMode::Passthrough; + let passthrough = config.interrupt_mode() == axvm_types::VMInterruptMode::Passthrough; Ok(ArmVcpuSetupConfig { passthrough_interrupt: passthrough, passthrough_timer: passthrough, diff --git a/virtualization/axvm/src/arch/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/mod.rs index fa0c8bcb89..afaee35950 100644 --- a/virtualization/axvm/src/arch/loongarch64/mod.rs +++ b/virtualization/axvm/src/arch/loongarch64/mod.rs @@ -17,7 +17,7 @@ use loongarch_vcpu::{ use super::{ ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, VcpuSetupContext, + VcpuRunAction, }; use crate::host::{HostMemory, HostTime, default_host}; @@ -76,17 +76,16 @@ impl ArchOps for LoongArch64Arch { } fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, + config: &crate::config::AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], ) -> AxResult<::SetupConfig> { - let (interrupt_mode, _console, _ports, _memory_regions, firmware_boot) = - ctx.into_parts(); - let passthrough = interrupt_mode == axvm_types::VMInterruptMode::Passthrough; + let passthrough = config.interrupt_mode() == axvm_types::VMInterruptMode::Passthrough; Ok(LoongArchVCpuSetupConfig { passthrough_interrupt: passthrough, passthrough_timer: passthrough, boot_args: [0; 3], boot_stack_top: 0, - firmware_boot, + firmware_boot: config.uses_firmware_boot(), }) } diff --git a/virtualization/axvm/src/arch/riscv64/mod.rs b/virtualization/axvm/src/arch/riscv64/mod.rs index eff1163d55..b0c7fcc531 100644 --- a/virtualization/axvm/src/arch/riscv64/mod.rs +++ b/virtualization/axvm/src/arch/riscv64/mod.rs @@ -16,7 +16,7 @@ use riscv_vplic::host::RiscvVplicHostIf; use super::{ ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, VcpuSetupContext, + VcpuRunAction, }; use crate::{ StopReason, @@ -108,9 +108,9 @@ impl ArchOps for Riscv64Arch { } fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, + _config: &crate::config::AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], ) -> AxResult<::SetupConfig> { - let _ = ctx.into_parts(); Ok(()) } diff --git a/virtualization/axvm/src/arch/x86_64/mod.rs b/virtualization/axvm/src/arch/x86_64/mod.rs index e519831cf6..9d8575efbb 100644 --- a/virtualization/axvm/src/arch/x86_64/mod.rs +++ b/virtualization/axvm/src/arch/x86_64/mod.rs @@ -24,7 +24,7 @@ use x86_vlapic::{ use super::{ ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, VcpuSetupContext, + VcpuRunAction, }; use crate::{ StopReason, @@ -81,17 +81,14 @@ impl ArchOps for X86_64Arch { } fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, + vm_config: &crate::config::AxVMConfig, + memory_regions: &[crate::vm::VMMemoryRegion], ) -> AxResult<::SetupConfig> { - let ( - _interrupt_mode, - emulates_console, - passthrough_ports, - memory_regions, - _firmware_boot, - ) = ctx.into_parts(); - let mut config = X86VCpuSetupConfig { - emulate_com1: emulates_console, + let mut setup_config = X86VCpuSetupConfig { + emulate_com1: vm_config + .emu_devices() + .iter() + .any(|device| device.emu_type == EmulatedDeviceType::Console), guest_memory_regions: memory_regions .iter() .map(|region| x86_vcpu::X86GuestMemoryRegion { @@ -102,10 +99,10 @@ impl ArchOps for X86_64Arch { .collect(), ..Default::default() }; - for port in passthrough_ports { - x86_result(config.add_passthrough_port_range(port.base, port.length))?; + for port in vm_config.pass_through_ports() { + x86_result(setup_config.add_passthrough_port_range(port.base, port.length))?; } - Ok(config) + Ok(setup_config) } fn new_nested_page_table(levels: usize) -> AxResult { diff --git a/virtualization/axvm/src/architecture/mod.rs b/virtualization/axvm/src/architecture/mod.rs index 8604f7770d..926d40051d 100644 --- a/virtualization/axvm/src/architecture/mod.rs +++ b/virtualization/axvm/src/architecture/mod.rs @@ -12,5 +12,4 @@ pub(crate) use exit::{handle_hypercall, handle_mmio_read, handle_mmio_write}; pub(crate) use ops::ArchOps; pub(crate) use types::{ BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, VcpuRunAction, - VcpuSetupContext, }; diff --git a/virtualization/axvm/src/architecture/ops.rs b/virtualization/axvm/src/architecture/ops.rs index 5774240fc4..4c2341bc6a 100644 --- a/virtualization/axvm/src/architecture/ops.rs +++ b/virtualization/axvm/src/architecture/ops.rs @@ -7,7 +7,7 @@ use ax_memory_addr::{PhysAddr, VirtAddr}; use axaddrspace::NestedPageTableOps; use axvm_types::{NestedPagingConfig, VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState}; -use super::{BoundVcpuExit, VcpuCreateContext, VcpuRunAction, VcpuSetupContext}; +use super::{BoundVcpuExit, VcpuCreateContext, VcpuRunAction}; pub(crate) trait ArchOps { type VCpu: VmArchVcpuOps; @@ -60,7 +60,8 @@ pub(crate) trait ArchOps { ) -> AxResult<::CreateConfig>; fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, + config: &crate::config::AxVMConfig, + memory_regions: &[crate::vm::VMMemoryRegion], ) -> AxResult<::SetupConfig>; fn register_platform_irq_injector() {} diff --git a/virtualization/axvm/src/architecture/types.rs b/virtualization/axvm/src/architecture/types.rs index d48b8d5d48..10fbd71030 100644 --- a/virtualization/axvm/src/architecture/types.rs +++ b/virtualization/axvm/src/architecture/types.rs @@ -1,6 +1,6 @@ //! Architecture-neutral vCPU contexts and normalized runtime actions. -use axvm_types::{AccessWidth, GuestPhysAddr, PassThroughPortConfig, VMInterruptMode}; +use axvm_types::{AccessWidth, GuestPhysAddr}; use crate::StopReason; @@ -61,31 +61,3 @@ impl VcpuCreateContext { ) } } - -pub(crate) struct VcpuSetupContext<'a> { - pub(crate) interrupt_mode: VMInterruptMode, - pub(crate) emulates_console: bool, - pub(crate) passthrough_ports: &'a [PassThroughPortConfig], - pub(crate) memory_regions: &'a [crate::vm::VMMemoryRegion], - pub(crate) firmware_boot: bool, -} - -impl<'a> VcpuSetupContext<'a> { - pub(crate) fn into_parts( - self, - ) -> ( - VMInterruptMode, - bool, - &'a [PassThroughPortConfig], - &'a [crate::vm::VMMemoryRegion], - bool, - ) { - ( - self.interrupt_mode, - self.emulates_console, - self.passthrough_ports, - self.memory_regions, - self.firmware_boot, - ) - } -} diff --git a/virtualization/axvm/src/config.rs b/virtualization/axvm/src/config.rs index 46aca8d0a2..264a1ebac2 100644 --- a/virtualization/axvm/src/config.rs +++ b/virtualization/axvm/src/config.rs @@ -240,6 +240,15 @@ impl AxVMConfig { self.boot_policy } + pub(crate) fn uses_firmware_boot(&self) -> bool { + matches!( + self.boot_policy, + GuestBootPolicy::AdjustKernelForBootProtocol { + protocol: VMBootProtocol::Uefi, + } + ) + } + /// Sets the policy used to adjust runtime boot image addresses. pub fn set_boot_policy(&mut self, boot_policy: GuestBootPolicy) { self.boot_policy = boot_policy; @@ -426,4 +435,16 @@ mod tests { assert_eq!(regions[1].size, 0x10000); assert_eq!(regions[1].map_type, VmMemMappingType::MapReserved); } + + #[test] + fn firmware_boot_is_derived_from_the_typed_boot_policy() { + let mut config = AxVMConfig::default_for_test(1, "linux"); + assert!(!config.uses_firmware_boot()); + + config.set_boot_policy(GuestBootPolicy::AdjustKernelForBootProtocol { + protocol: VMBootProtocol::Uefi, + }); + + assert!(config.uses_firmware_boot()); + } } diff --git a/virtualization/axvm/src/vm.rs b/virtualization/axvm/src/vm/mod.rs similarity index 99% rename from virtualization/axvm/src/vm.rs rename to virtualization/axvm/src/vm/mod.rs index fadc984ef9..01e427e6ef 100644 --- a/virtualization/axvm/src/vm.rs +++ b/virtualization/axvm/src/vm/mod.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Virtual machine state, resources, and lifecycle entry points. + use alloc::{boxed::Box, collections::BTreeMap, format, string::String, sync::Arc, vec::Vec}; use core::{ alloc::Layout, diff --git a/virtualization/axvm/src/vm/prepare/vcpus.rs b/virtualization/axvm/src/vm/prepare/vcpus.rs index d3081e4641..f2e948cacc 100644 --- a/virtualization/axvm/src/vm/prepare/vcpus.rs +++ b/virtualization/axvm/src/vm/prepare/vcpus.rs @@ -3,13 +3,10 @@ use alloc::{boxed::Box, sync::Arc, vec::Vec}; use ax_errno::AxResult; -use axvm_types::{EmulatedDeviceType, GuestPhysAddr}; +use axvm_types::GuestPhysAddr; use super::super::{AxVCpuRef, AxVMResources, VCpu}; -use crate::{ - arch::{ArchOps, CurrentArch, VcpuCreateContext, VcpuSetupContext}, - config::{GuestBootPolicy, VMBootProtocol}, -}; +use crate::arch::{ArchOps, CurrentArch, VcpuCreateContext}; pub(super) struct PreparedVcpus { vcpus: Vec, @@ -23,7 +20,7 @@ impl PreparedVcpus { ) -> AxResult { let vcpu_id_pcpu_sets = resources.config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); let create_state = CurrentArch::new_vcpu_create_state(&vcpu_id_pcpu_sets)?; - let firmware_boot = guest_uses_firmware_boot(resources); + let firmware_boot = resources.config.uses_firmware_boot(); debug!("dtb_load_gpa: {dtb_addr:?}"); debug!("id: {vm_id}, VCpuIdPCpuSets: {vcpu_id_pcpu_sets:#x?}"); @@ -57,17 +54,8 @@ impl PreparedVcpus { pub(super) fn setup(&self, resources: &AxVMResources) -> AxResult { for vcpu in &self.vcpus { - let setup_config = CurrentArch::build_vcpu_setup_config(VcpuSetupContext { - interrupt_mode: resources.config.interrupt_mode(), - emulates_console: resources - .config - .emu_devices() - .iter() - .any(|dev| dev.emu_type == EmulatedDeviceType::Console), - passthrough_ports: resources.config.pass_through_ports(), - memory_regions: &resources.memory_regions, - firmware_boot: guest_uses_firmware_boot(resources), - })?; + let setup_config = + CurrentArch::build_vcpu_setup_config(&resources.config, &resources.memory_regions)?; let entry = if vcpu.id() == 0 { resources.config.bsp_entry() @@ -86,12 +74,3 @@ impl PreparedVcpus { self.vcpus.into_boxed_slice() } } - -fn guest_uses_firmware_boot(resources: &AxVMResources) -> bool { - matches!( - resources.config.boot_policy(), - GuestBootPolicy::AdjustKernelForBootProtocol { - protocol: VMBootProtocol::Uefi, - } - ) -} diff --git a/virtualization/axvm/tests/arch_boundary_contract.rs b/virtualization/axvm/tests/arch_boundary_contract.rs index 99ea20289e..40546a3833 100644 --- a/virtualization/axvm/tests/arch_boundary_contract.rs +++ b/virtualization/axvm/tests/arch_boundary_contract.rs @@ -14,7 +14,7 @@ #[test] fn vm_core_does_not_handle_arch_local_exits() { - let vm_rs = include_str!("../src/vm.rs"); + let vm_rs = include_str!("../src/vm/mod.rs"); for forbidden in [ "CurrentArch", @@ -25,7 +25,7 @@ fn vm_core_does_not_handle_arch_local_exits() { ] { assert!( !vm_rs.contains(forbidden), - "vm.rs must not contain architecture-local exit handling detail: {forbidden}" + "vm/mod.rs must not contain architecture-local exit handling detail: {forbidden}" ); } } @@ -116,6 +116,20 @@ fn common_domains_live_outside_architecture_directories() { } } +#[test] +fn vm_domain_uses_a_canonical_directory_module() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + + assert!( + source_root.join("vm/mod.rs").is_file(), + "the VM domain with child modules must use vm/mod.rs as its directory page" + ); + assert!( + !source_root.join("vm.rs").exists(), + "vm.rs must not coexist with the vm child-module directory" + ); +} + #[test] fn common_modules_do_not_include_architecture_sources() { let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); @@ -198,6 +212,20 @@ fn host_time_trait_only_exposes_common_clock_capabilities() { } } +#[test] +fn vcpu_setup_context_keeps_named_capabilities() { + let types = include_str!("../src/architecture/types.rs"); + let ops = include_str!("../src/architecture/ops.rs"); + let preparation = include_str!("../src/vm/prepare/vcpus.rs"); + + assert!( + [&types, &ops, &preparation] + .into_iter() + .all(|source| !source.contains("VcpuSetupContext")), + "vCPU setup must pass named configuration and memory sources without a union context" + ); +} + fn find_target_arch_cfg_outside_arch( source_root: &std::path::Path, directory: &std::path::Path, From 543d03c2bfd61e6a4fabb82840b2d16d06a1b4b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Mon, 13 Jul 2026 11:43:22 +0800 Subject: [PATCH 5/5] refactor(axvm): centralize architecture VM initialization --- os/axvisor/src/shell/command/vm.rs | 7 +- .../axvm/src/arch/aarch64/capabilities.rs | 53 +----- virtualization/axvm/src/arch/aarch64/mod.rs | 95 +--------- virtualization/axvm/src/arch/aarch64/vm.rs | 177 ++++++++++++++++++ .../axvm/src/arch/loongarch64/capabilities.rs | 8 +- .../axvm/src/arch/loongarch64/mod.rs | 58 +----- .../axvm/src/arch/loongarch64/vm.rs | 128 +++++++++++++ virtualization/axvm/src/arch/mod.rs | 52 +---- .../axvm/src/arch/riscv64/capabilities.rs | 16 +- virtualization/axvm/src/arch/riscv64/mod.rs | 65 +------ virtualization/axvm/src/arch/riscv64/vm.rs | 110 +++++++++++ .../axvm/src/arch/x86_64/capabilities.rs | 62 +----- virtualization/axvm/src/arch/x86_64/mod.rs | 52 +---- virtualization/axvm/src/arch/x86_64/vm.rs | 173 +++++++++++++++++ .../axvm/src/architecture/capabilities.rs | 32 ---- virtualization/axvm/src/architecture/mod.rs | 8 +- virtualization/axvm/src/architecture/ops.rs | 51 +---- virtualization/axvm/src/architecture/types.rs | 18 -- virtualization/axvm/src/config.rs | 21 --- virtualization/axvm/src/lifecycle/machine.rs | 70 +------ virtualization/axvm/src/lifecycle/status.rs | 3 - virtualization/axvm/src/vm/mod.rs | 74 +++----- virtualization/axvm/src/vm/prepare.rs | 147 +++++++++------ .../axvm/src/vm/prepare/address_space.rs | 12 +- virtualization/axvm/src/vm/prepare/devices.rs | 19 +- virtualization/axvm/src/vm/prepare/vcpus.rs | 83 ++++---- .../axvm/tests/arch_boundary_contract.rs | 117 +++++++++++- 27 files changed, 912 insertions(+), 799 deletions(-) create mode 100644 virtualization/axvm/src/arch/aarch64/vm.rs create mode 100644 virtualization/axvm/src/arch/loongarch64/vm.rs create mode 100644 virtualization/axvm/src/arch/riscv64/vm.rs create mode 100644 virtualization/axvm/src/arch/x86_64/vm.rs diff --git a/os/axvisor/src/shell/command/vm.rs b/os/axvisor/src/shell/command/vm.rs index 041130b187..75faf258c6 100644 --- a/os/axvisor/src/shell/command/vm.rs +++ b/os/axvisor/src/shell/command/vm.rs @@ -34,7 +34,6 @@ fn can_start_vm(status: VmStatus) -> Result<(), &'static str> { VmStatus::Running => Err("VM is already running"), VmStatus::Paused => Err("VM is suspended, use 'vm resume' instead"), VmStatus::Stopping => Err("VM is stopping, wait for it to fully stop"), - VmStatus::Uninit => Err("VM is still loading"), VmStatus::Pausing => Err("VM is pausing"), VmStatus::Destroying | VmStatus::Destroyed => Err("VM is being destroyed"), VmStatus::Failed => Err("VM is failed"), @@ -54,7 +53,7 @@ fn can_stop_vm(status: VmStatus, force: bool) -> Result<(), &'static str> { } } VmStatus::Stopped => Err("VM is already stopped"), - VmStatus::Uninit | VmStatus::Ready => Ok(()), // Allow stopping VMs in these states + VmStatus::Ready => Ok(()), // Allow stopping VMs before their first start. VmStatus::Pausing => Err("VM is pausing"), VmStatus::Destroying | VmStatus::Destroyed => Err("VM is being destroyed"), VmStatus::Failed => Err("VM is failed"), @@ -68,7 +67,6 @@ fn can_suspend_vm(status: VmStatus) -> Result<(), &'static str> { VmStatus::Paused => Err("VM is already suspended"), VmStatus::Stopped => Err("VM is stopped, cannot suspend"), VmStatus::Stopping => Err("VM is stopping, cannot suspend"), - VmStatus::Uninit => Err("VM is loading, cannot suspend"), VmStatus::Ready => Err("VM is not running, cannot suspend"), VmStatus::Pausing => Err("VM is already pausing"), VmStatus::Destroying | VmStatus::Destroyed => Err("VM is being destroyed"), @@ -83,7 +81,6 @@ fn can_resume_vm(status: VmStatus) -> Result<(), &'static str> { VmStatus::Running => Err("VM is already running"), VmStatus::Stopped => Err("VM is stopped, use 'vm start' instead"), VmStatus::Stopping => Err("VM is stopping, cannot resume"), - VmStatus::Uninit => Err("VM is loading, cannot resume"), VmStatus::Ready => Err("VM is not started yet, use 'vm start' instead"), VmStatus::Pausing => Err("VM is pausing, wait before resuming"), VmStatus::Destroying | VmStatus::Destroyed => Err("VM is being destroyed"), @@ -305,7 +302,7 @@ fn stop_vm_by_id(vm_id: usize, force: bool) { println!("Gracefully stopping VM[{}]...", vm_id); } } - VmStatus::Uninit | VmStatus::Ready => { + VmStatus::Ready => { println!( "âš  VM[{}] is in {:?} state, stopping anyway...", vm_id, status diff --git a/virtualization/axvm/src/arch/aarch64/capabilities.rs b/virtualization/axvm/src/arch/aarch64/capabilities.rs index ca370c3731..b232555ad7 100644 --- a/virtualization/axvm/src/arch/aarch64/capabilities.rs +++ b/virtualization/axvm/src/arch/aarch64/capabilities.rs @@ -1,32 +1,11 @@ //! AArch64 implementations of AxVM platform capability hooks. -use alloc::{format, sync::Arc}; +use alloc::format; use ax_errno::{AxResult, ax_err_type}; -use axdevice_base::DeviceRegistry as _; -use axvm_types::VMInterruptMode; use super::Aarch64Arch; -use crate::architecture::{ - AddressSpacePlatform, BootImagePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, -}; - -impl DevicePlatform for Aarch64Arch { - fn register_devices( - vm: &crate::AxVM, - config: &crate::config::AxVMConfig, - devices: &mut axdevice::AxVmDevices, - ) -> AxResult { - if config.interrupt_mode() == VMInterruptMode::Passthrough { - assign_passthrough_spis(vm, config, devices); - } else { - register_virtual_timers(devices)?; - } - Ok(()) - } -} - -impl AddressSpacePlatform for Aarch64Arch {} +use crate::architecture::{BootImagePlatform, GuestBootPlatform, HostTimePlatform}; impl HostTimePlatform for Aarch64Arch {} @@ -105,31 +84,3 @@ pub(super) fn patch_provided_fdt( })?; super::fdt::update_cpu_node(&provided_fdt, host_fdt.as_ref(), crate_config) } - -fn assign_passthrough_spis( - vm: &crate::AxVM, - config: &crate::config::AxVMConfig, - devices: &axdevice::AxVmDevices, -) { - let cpu_id = vm.id() - 1; // FIXME: get the real CPU id. - let Some(gicd) = devices - .devices() - .find_map(|device| device.as_any().downcast_ref::()) - else { - warn!("Failed to assign SPIs: No VGicD found in device list"); - return; - }; - - for spi in config.pass_through_spis() { - gicd.assign_irq(*spi + 32, cpu_id, (0, 0, 0, cpu_id as _)); - } -} - -fn register_virtual_timers(devices: &mut axdevice::AxVmDevices) -> AxResult { - for device in axdevice::create_vtimer_devices() { - devices - .register(Arc::from(device) as Arc) - .map_err(|err| ax_err_type!(InvalidInput, format!("register vtimer: {err:?}")))?; - } - Ok(()) -} diff --git a/virtualization/axvm/src/arch/aarch64/mod.rs b/virtualization/axvm/src/arch/aarch64/mod.rs index 5bbcedbb34..702663fb88 100644 --- a/virtualization/axvm/src/arch/aarch64/mod.rs +++ b/virtualization/axvm/src/arch/aarch64/mod.rs @@ -20,14 +20,8 @@ use axvm_types::{ VmArchVcpuOps, }; -use super::{ - ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, -}; -use crate::{ - architecture::ops::target_phys_cpu_ids, - host::{HostCpu, HostMemory, HostTime, default_host}, -}; +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; +use crate::host::{HostCpu, HostMemory, HostTime, default_host}; mod capabilities; #[path = "../../architecture/cpu_up.rs"] @@ -39,6 +33,7 @@ mod ipi; mod npt; #[path = "../../architecture/sysreg.rs"] mod sysreg; +mod vm; pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; use cpu_up::{CpuUpExit, CpuUpOps}; @@ -58,7 +53,6 @@ impl CpuUpOps for Aarch64Arch {} impl ArchOps for Aarch64Arch { type VCpu = AxvmArmVcpu; type PerCpu = AxvmArmPerCpu; - type VcpuCreateState = (); type DeferredRunWork = Aarch64DeferredRunWork; type NestedPageTable = npt::NestedPageTable; @@ -66,61 +60,6 @@ impl ArchOps for Aarch64Arch { arm_vcpu::has_hardware_support() } - fn max_guest_page_table_levels() -> usize { - arm_vcpu::max_guest_page_table_levels() - } - - fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { - let mut selected = usize::MAX; - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - let levels = crate::percpu::cpu_max_guest_page_table_levels(cpu_id) - .unwrap_or_else(arm_vcpu::max_guest_page_table_levels); - if levels == 0 { - return ax_err!( - Unsupported, - "AArch64 nested paging is not enabled on target CPU" - ); - } - selected = selected.min(levels); - } - if selected == usize::MAX { - selected = arm_vcpu::max_guest_page_table_levels(); - } - match selected { - 3 | 4 => Ok(selected), - _ => ax_err!(Unsupported, "unsupported AArch64 stage-2 page-table levels"), - } - } - - fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - let mut pa_bits = usize::MAX; - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - let bits = - crate::percpu::cpu_guest_phys_addr_bits(cpu_id).unwrap_or_else(arm_vcpu::pa_bits); - pa_bits = pa_bits.min(bits); - } - if pa_bits == usize::MAX { - pa_bits = arm_vcpu::pa_bits(); - } - - let gpa_bits = match levels { - 3 => 39, - 4 => 48, - _ => return ax_err!(InvalidInput, "unsupported AArch64 stage-2 levels"), - }; - Ok(NestedPagingConfig::new( - root_paddr, levels, gpa_bits, pa_bits, - )) - } - - fn new_nested_page_table(levels: usize) -> AxResult { - npt::NestedPageTable::new(levels) - } - fn clean_dcache_range(addr: VirtAddr, size: usize) { aarch64_cpu_ext::cache::dcache_range( aarch64_cpu_ext::cache::CacheOp::Clean, @@ -129,34 +68,6 @@ impl ArchOps for Aarch64Arch { ); } - fn new_vcpu_create_state( - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - Ok(()) - } - - fn build_vcpu_create_config( - _state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig> { - let (_vcpu_id, phys_cpu_id, dtb_addr, _firmware_boot) = ctx.into_parts(); - Ok(ArmVcpuCreateConfig { - mpidr_el1: phys_cpu_id as _, - dtb_addr: dtb_addr.unwrap_or_default().as_usize(), - }) - } - - fn build_vcpu_setup_config( - config: &crate::config::AxVMConfig, - _memory_regions: &[crate::vm::VMMemoryRegion], - ) -> AxResult<::SetupConfig> { - let passthrough = config.interrupt_mode() == axvm_types::VMInterruptMode::Passthrough; - Ok(ArmVcpuSetupConfig { - passthrough_interrupt: passthrough, - passthrough_timer: passthrough, - }) - } - fn handle_vcpu_exit_bound( vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef, diff --git a/virtualization/axvm/src/arch/aarch64/vm.rs b/virtualization/axvm/src/arch/aarch64/vm.rs new file mode 100644 index 0000000000..43b3972f18 --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/vm.rs @@ -0,0 +1,177 @@ +//! AArch64 VM resource creation and initialization. + +use alloc::sync::Arc; + +use arm_vcpu::{ArmVcpuCreateConfig, ArmVcpuSetupConfig}; +use ax_errno::{AxResult, ax_err, ax_err_type}; +use axdevice_base::DeviceRegistry as _; +use axvm_types::{NestedPagingConfig, VMInterruptMode, VmArchVcpuOps}; + +use super::{Aarch64Arch, npt}; +use crate::{ + config::AxVMConfig, + vm::{ + AxVM, AxVMResources, + prepare::{ + PreparedVm, VmInitRequest, + address_space::{guest_owned_regions, map_guest_address_space}, + complete_vm_init, default_device_factories, + devices::PreparedDevices, + validate_guest_dtb, + vcpus::{PreparedVcpus, vcpu_placements}, + }, + }, +}; + +impl Aarch64Arch { + pub(crate) fn create_vm_resources(config: AxVMConfig) -> AxResult { + let placements = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); + let levels = guest_page_table_levels(&placements)?; + let page_table = npt::NestedPageTable::new(levels)?; + AxVMResources::from_page_table(config, page_table, |root_paddr| { + nested_paging_config(root_paddr, levels, &placements) + }) + } + + pub(crate) fn init_vm(vm: &AxVM, request: VmInitRequest<'_>) -> AxResult { + match request { + VmInitRequest::Default => { + let factories = default_device_factories()?; + let interrupt_fabric = crate::InterruptFabric::new(vm.interrupt_mode()); + init_vm_with(vm, &factories, interrupt_fabric) + } + VmInitRequest::Provided { + factories, + interrupt_fabric, + } => init_vm_with(vm, factories, interrupt_fabric), + } + } +} + +fn init_vm_with( + vm: &AxVM, + factories: &axdevice::DeviceFactoryRegistry, + interrupt_fabric: crate::InterruptFabric, +) -> AxResult { + complete_vm_init(vm, interrupt_fabric, |resources, interrupt_fabric| { + let placements = vcpu_placements(resources); + let dtb_addr = resources + .config() + .image_config() + .dtb_load_gpa + .unwrap_or_default(); + let vcpus = PreparedVcpus::create(vm.id(), &placements, |placement| { + Ok(ArmVcpuCreateConfig { + mpidr_el1: placement.phys_cpu_id as _, + dtb_addr: dtb_addr.as_usize(), + }) + })?; + let mut devices = PreparedDevices::build_common(resources, factories, interrupt_fabric)?; + register_arch_devices(vm, resources.config(), &mut devices.devices)?; + devices.register_special_devices(vm)?; + validate_guest_dtb(resources)?; + + let owned_regions = guest_owned_regions(resources); + map_guest_address_space(vm, resources, devices.devices(), &owned_regions)?; + vcpus.setup(resources, build_vcpu_setup_config)?; + + Ok(PreparedVm::new(vcpus, devices)) + }) +} + +fn build_vcpu_setup_config( + config: &AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], +) -> AxResult<::SetupConfig> { + let passthrough = config.interrupt_mode() == VMInterruptMode::Passthrough; + Ok(ArmVcpuSetupConfig { + passthrough_interrupt: passthrough, + passthrough_timer: passthrough, + }) +} + +fn register_arch_devices( + vm: &AxVM, + config: &AxVMConfig, + devices: &mut axdevice::AxVmDevices, +) -> AxResult { + if config.interrupt_mode() == VMInterruptMode::Passthrough { + assign_passthrough_spis(vm, config, devices); + } else { + register_virtual_timers(devices)?; + } + Ok(()) +} + +fn assign_passthrough_spis(vm: &AxVM, config: &AxVMConfig, devices: &axdevice::AxVmDevices) { + let cpu_id = vm.id() - 1; // FIXME: get the real CPU id. + let Some(gicd) = devices + .devices() + .find_map(|device| device.as_any().downcast_ref::()) + else { + warn!("Failed to assign SPIs: No VGicD found in device list"); + return; + }; + + for spi in config.pass_through_spis() { + gicd.assign_irq(*spi + 32, cpu_id, (0, 0, 0, cpu_id as _)); + } +} + +fn register_virtual_timers(devices: &mut axdevice::AxVmDevices) -> AxResult { + for device in axdevice::create_vtimer_devices() { + devices + .register(Arc::from(device) as Arc) + .map_err(|err| { + ax_err_type!(InvalidInput, alloc::format!("register vtimer: {err:?}")) + })?; + } + Ok(()) +} + +fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { + let mut selected = usize::MAX; + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + let levels = crate::percpu::cpu_max_guest_page_table_levels(cpu_id) + .unwrap_or_else(arm_vcpu::max_guest_page_table_levels); + if levels == 0 { + return ax_err!( + Unsupported, + "AArch64 nested paging is not enabled on target CPU" + ); + } + selected = selected.min(levels); + } + if selected == usize::MAX { + selected = arm_vcpu::max_guest_page_table_levels(); + } + match selected { + 3 | 4 => Ok(selected), + _ => ax_err!(Unsupported, "unsupported AArch64 stage-2 page-table levels"), + } +} + +fn nested_paging_config( + root_paddr: ax_memory_addr::PhysAddr, + levels: usize, + vcpu_mappings: &[(usize, Option, usize)], +) -> AxResult { + let mut pa_bits = usize::MAX; + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + let bits = + crate::percpu::cpu_guest_phys_addr_bits(cpu_id).unwrap_or_else(arm_vcpu::pa_bits); + pa_bits = pa_bits.min(bits); + } + if pa_bits == usize::MAX { + pa_bits = arm_vcpu::pa_bits(); + } + + let gpa_bits = match levels { + 3 => 39, + 4 => 48, + _ => return ax_err!(InvalidInput, "unsupported AArch64 stage-2 levels"), + }; + Ok(NestedPagingConfig::new( + root_paddr, levels, gpa_bits, pa_bits, + )) +} diff --git a/virtualization/axvm/src/arch/loongarch64/capabilities.rs b/virtualization/axvm/src/arch/loongarch64/capabilities.rs index 14472f98a3..cba7b09970 100644 --- a/virtualization/axvm/src/arch/loongarch64/capabilities.rs +++ b/virtualization/axvm/src/arch/loongarch64/capabilities.rs @@ -1,13 +1,7 @@ //! LoongArch64 implementations of AxVM platform capability hooks. use super::LoongArch64Arch; -use crate::architecture::{ - AddressSpacePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, -}; - -impl DevicePlatform for LoongArch64Arch {} - -impl AddressSpacePlatform for LoongArch64Arch {} +use crate::architecture::{GuestBootPlatform, HostTimePlatform}; impl GuestBootPlatform for LoongArch64Arch { fn init_guest_boot_resources() { diff --git a/virtualization/axvm/src/arch/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/mod.rs index afaee35950..b42a41f2c6 100644 --- a/virtualization/axvm/src/arch/loongarch64/mod.rs +++ b/virtualization/axvm/src/arch/loongarch64/mod.rs @@ -9,16 +9,12 @@ use axvm_types::{ }; use loongarch_vcpu::{ LoongArchAccessFlags, LoongArchAccessWidth, LoongArchGuestPhysAddr, LoongArchHostOps, - LoongArchHostPhysAddr, LoongArchHostVirtAddr, LoongArchIocsrStateRef, - LoongArchNestedPagingConfig, LoongArchPerCpu, LoongArchVCpuCreateConfig, - LoongArchVCpuSetupConfig, LoongArchVcpu, LoongArchVcpuError, LoongArchVcpuResult, - LoongArchVmExit, + LoongArchHostPhysAddr, LoongArchHostVirtAddr, LoongArchNestedPagingConfig, LoongArchPerCpu, + LoongArchVCpuCreateConfig, LoongArchVCpuSetupConfig, LoongArchVcpu, LoongArchVcpuError, + LoongArchVcpuResult, LoongArchVmExit, }; -use super::{ - ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, -}; +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; use crate::host::{HostMemory, HostTime, default_host}; pub(crate) mod boot; @@ -27,6 +23,7 @@ pub(crate) mod fdt; mod idle; pub(crate) mod irq; mod npt; +mod vm; pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; @@ -41,7 +38,6 @@ pub(crate) enum LoongArchDeferredRunWork { impl ArchOps for LoongArch64Arch { type VCpu = AxvmLoongArchVcpu; type PerCpu = AxvmLoongArchPerCpu; - type VcpuCreateState = LoongArchIocsrStateRef; type DeferredRunWork = LoongArchDeferredRunWork; type NestedPageTable = npt::NestedPageTable; @@ -49,50 +45,6 @@ impl ArchOps for LoongArch64Arch { loongarch_vcpu::has_hardware_support() } - fn new_vcpu_create_state( - vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - let vcpu_state_count = vcpu_mappings - .iter() - .map(|(vcpu_id, ..)| *vcpu_id) - .max() - .map_or(0, |vcpu_id| vcpu_id + 1); - loongarch_result(loongarch_vcpu::LoongArchIocsrState::new(vcpu_state_count)) - } - - fn build_vcpu_create_config( - state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig> { - let (vcpu_id, _phys_cpu_id, dtb_addr, firmware_boot) = ctx.into_parts(); - Ok(LoongArchVCpuCreateConfig { - cpu_id: vcpu_id, - dtb_addr: dtb_addr.unwrap_or_default().as_usize(), - boot_args: [0; 3], - boot_stack_top: 0, - firmware_boot, - iocsr_state: state.clone(), - }) - } - - fn build_vcpu_setup_config( - config: &crate::config::AxVMConfig, - _memory_regions: &[crate::vm::VMMemoryRegion], - ) -> AxResult<::SetupConfig> { - let passthrough = config.interrupt_mode() == axvm_types::VMInterruptMode::Passthrough; - Ok(LoongArchVCpuSetupConfig { - passthrough_interrupt: passthrough, - passthrough_timer: passthrough, - boot_args: [0; 3], - boot_stack_top: 0, - firmware_boot: config.uses_firmware_boot(), - }) - } - - fn new_nested_page_table(levels: usize) -> AxResult { - npt::NestedPageTable::new(levels) - } - fn register_platform_irq_injector() { irq::register_platform_irq_injector(); } diff --git a/virtualization/axvm/src/arch/loongarch64/vm.rs b/virtualization/axvm/src/arch/loongarch64/vm.rs new file mode 100644 index 0000000000..0c78c57974 --- /dev/null +++ b/virtualization/axvm/src/arch/loongarch64/vm.rs @@ -0,0 +1,128 @@ +//! LoongArch64 VM resource creation and initialization. + +use ax_errno::AxResult; +use axvm_types::{NestedPagingConfig, VmArchVcpuOps}; +use loongarch_vcpu::{LoongArchVCpuCreateConfig, LoongArchVCpuSetupConfig}; + +use super::{LoongArch64Arch, loongarch_result, npt}; +use crate::{ + config::AxVMConfig, + vm::{ + AxVM, AxVMResources, + prepare::{ + PreparedVm, VmInitRequest, + address_space::{guest_owned_regions, map_guest_address_space}, + complete_vm_init, default_device_factories, + devices::PreparedDevices, + validate_guest_dtb, + vcpus::{PreparedVcpus, vcpu_placements}, + }, + }, +}; + +impl LoongArch64Arch { + pub(crate) fn create_vm_resources(config: AxVMConfig) -> AxResult { + let placements = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); + let levels = guest_page_table_levels(&placements); + let page_table = npt::NestedPageTable::new(levels)?; + AxVMResources::from_page_table(config, page_table, |root_paddr| { + let gpa_bits = match levels { + 3 => 39, + 4 => 48, + _ => { + return ax_errno::ax_err!( + InvalidInput, + "unsupported LoongArch nested page-table levels" + ); + } + }; + Ok(NestedPagingConfig::new(root_paddr, levels, gpa_bits, 0)) + }) + } + + pub(crate) fn init_vm(vm: &AxVM, request: VmInitRequest<'_>) -> AxResult { + match request { + VmInitRequest::Default => { + let factories = default_device_factories()?; + let interrupt_fabric = crate::InterruptFabric::new(vm.interrupt_mode()); + init_vm_with(vm, &factories, interrupt_fabric) + } + VmInitRequest::Provided { + factories, + interrupt_fabric, + } => init_vm_with(vm, factories, interrupt_fabric), + } + } +} + +fn init_vm_with( + vm: &AxVM, + factories: &axdevice::DeviceFactoryRegistry, + interrupt_fabric: crate::InterruptFabric, +) -> AxResult { + complete_vm_init(vm, interrupt_fabric, |resources, interrupt_fabric| { + let placements = vcpu_placements(resources); + let state_count = placements + .iter() + .map(|placement| placement.id) + .max() + .map_or(0, |vcpu_id| vcpu_id + 1); + let iocsr_state = loongarch_result(loongarch_vcpu::LoongArchIocsrState::new(state_count))?; + let dtb_addr = resources + .config() + .image_config() + .dtb_load_gpa + .unwrap_or_default(); + let firmware_boot = uses_firmware_boot(resources.config()); + let vcpus = PreparedVcpus::create(vm.id(), &placements, |placement| { + Ok(LoongArchVCpuCreateConfig { + cpu_id: placement.id, + dtb_addr: dtb_addr.as_usize(), + boot_args: [0; 3], + boot_stack_top: 0, + firmware_boot, + iocsr_state: iocsr_state.clone(), + }) + })?; + let mut devices = PreparedDevices::build_common(resources, factories, interrupt_fabric)?; + devices.register_special_devices(vm)?; + validate_guest_dtb(resources)?; + + let owned_regions = guest_owned_regions(resources); + map_guest_address_space(vm, resources, devices.devices(), &owned_regions)?; + vcpus.setup(resources, build_vcpu_setup_config)?; + + Ok(PreparedVm::new(vcpus, devices)) + }) +} + +fn build_vcpu_setup_config( + config: &AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], +) -> AxResult<::SetupConfig> { + let passthrough = config.interrupt_mode() == axvm_types::VMInterruptMode::Passthrough; + Ok(LoongArchVCpuSetupConfig { + passthrough_interrupt: passthrough, + passthrough_timer: passthrough, + boot_args: [0; 3], + boot_stack_top: 0, + firmware_boot: uses_firmware_boot(config), + }) +} + +fn uses_firmware_boot(config: &AxVMConfig) -> bool { + matches!( + config.boot_policy(), + crate::config::GuestBootPolicy::AdjustKernelForBootProtocol { + protocol: crate::config::VMBootProtocol::Uefi, + } + ) +} + +fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> usize { + let mut levels = 4; + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + levels = levels.min(crate::percpu::cpu_max_guest_page_table_levels(cpu_id).unwrap_or(4)); + } + levels +} diff --git a/virtualization/axvm/src/arch/mod.rs b/virtualization/axvm/src/arch/mod.rs index cf445c9b5f..c1db6f0739 100644 --- a/virtualization/axvm/src/arch/mod.rs +++ b/virtualization/axvm/src/arch/mod.rs @@ -1,13 +1,9 @@ //! Target architecture selection and stable internal dispatch. use ax_errno::AxResult; -use ax_memory_addr::PhysAddr; -use axvm_types::NestedPagingConfig; pub(crate) use crate::architecture::*; -use crate::architecture::{ - AddressSpacePlatform, BootImagePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, -}; +use crate::architecture::{BootImagePlatform, GuestBootPlatform, HostTimePlatform}; #[cfg(target_arch = "aarch64")] mod aarch64; @@ -80,52 +76,6 @@ pub(crate) type ArchVCpu = ::VCpu; pub(crate) type ArchPerCpu = ::PerCpu; pub(crate) type ArchNestedPageTable = ::NestedPageTable; -pub(crate) fn guest_page_table_levels( - vcpu_mappings: &[(usize, Option, usize)], -) -> AxResult { - CurrentArch::guest_page_table_levels(vcpu_mappings) -} - -pub(crate) fn new_nested_page_table(levels: usize) -> AxResult { - CurrentArch::new_nested_page_table(levels) -} - -pub(crate) fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - vcpu_mappings: &[(usize, Option, usize)], -) -> AxResult { - CurrentArch::nested_paging_config(root_paddr, levels, vcpu_mappings) -} - -pub(crate) fn configure_interrupt_fabric( - factories: &mut axdevice::DeviceFactoryRegistry, - mode: axvm_types::VMInterruptMode, - configs: &[axvm_types::EmulatedDeviceConfig], -) -> AxResult { - CurrentArch::configure_interrupt_fabric(factories, mode, configs) -} - -pub(crate) fn register_arch_devices( - vm: &crate::AxVM, - config: &crate::config::AxVMConfig, - devices: &mut axdevice::AxVmDevices, -) -> AxResult { - CurrentArch::register_devices(vm, config, devices) -} - -pub(crate) fn append_arch_owned_regions( - regions: &mut alloc::vec::Vec, -) { - CurrentArch::append_owned_regions(regions); -} - -pub(crate) fn map_arch_address_space( - address_space: &mut axaddrspace::AddrSpace, -) -> AxResult { - CurrentArch::map_address_space(address_space) -} - pub(crate) fn register_timer_callback() { CurrentArch::register_timer_callback(); } diff --git a/virtualization/axvm/src/arch/riscv64/capabilities.rs b/virtualization/axvm/src/arch/riscv64/capabilities.rs index 9d26752f75..70f2a1700e 100644 --- a/virtualization/axvm/src/arch/riscv64/capabilities.rs +++ b/virtualization/axvm/src/arch/riscv64/capabilities.rs @@ -5,21 +5,7 @@ use alloc::{format, vec::Vec}; use ax_errno::{AxResult, ax_err_type}; use super::Riscv64Arch; -use crate::architecture::{ - AddressSpacePlatform, BootImagePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, -}; - -impl DevicePlatform for Riscv64Arch { - fn configure_interrupt_fabric( - factories: &mut axdevice::DeviceFactoryRegistry, - mode: axvm_types::VMInterruptMode, - configs: &[axvm_types::EmulatedDeviceConfig], - ) -> AxResult { - super::irq::configure(factories, mode, configs) - } -} - -impl AddressSpacePlatform for Riscv64Arch {} +use crate::architecture::{BootImagePlatform, GuestBootPlatform, HostTimePlatform}; impl HostTimePlatform for Riscv64Arch {} diff --git a/virtualization/axvm/src/arch/riscv64/mod.rs b/virtualization/axvm/src/arch/riscv64/mod.rs index b0c7fcc531..76ab6c6b0a 100644 --- a/virtualization/axvm/src/arch/riscv64/mod.rs +++ b/virtualization/axvm/src/arch/riscv64/mod.rs @@ -1,7 +1,7 @@ use alloc::vec::Vec; use ax_crate_interface::impl_interface; -use ax_errno::{AxError, AxResult, ax_err}; +use ax_errno::{AxError, AxResult}; use ax_memory_addr::{PhysAddr, VirtAddr}; use axvm_types::{ AccessWidth, GuestPhysAddr, MappingFlags, NestedPagingConfig, VCpuId, VMId, VMInterruptMode, @@ -14,13 +14,10 @@ use riscv_vcpu::{ }; use riscv_vplic::host::RiscvVplicHostIf; -use super::{ - ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, -}; +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; use crate::{ StopReason, - architecture::ops::{default_vcpu_affinities, target_phys_cpu_ids}, + architecture::ops::default_vcpu_affinities, host::{HostMemory, default_host}, }; @@ -31,6 +28,7 @@ pub(crate) mod fdt; mod images; mod irq; mod npt; +mod vm; pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; use cpu_up::{CpuUpExit, CpuUpOps}; @@ -52,7 +50,6 @@ impl CpuUpOps for Riscv64Arch { impl ArchOps for Riscv64Arch { type VCpu = AxvmRiscvVcpu; type PerCpu = AxvmRiscvPerCpu; - type VcpuCreateState = (); type DeferredRunWork = RiscvDeferredRunWork; type NestedPageTable = npt::NestedPageTable; @@ -60,60 +57,6 @@ impl ArchOps for Riscv64Arch { riscv_vcpu::has_hardware_support() } - fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { - let mut levels = riscv_vcpu::max_guest_page_table_levels(); - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - levels = levels.min( - crate::percpu::cpu_max_guest_page_table_levels(cpu_id) - .unwrap_or_else(riscv_vcpu::max_guest_page_table_levels), - ); - } - match levels { - 3 | 4 => Ok(levels), - _ => ax_err!(Unsupported, "no supported RISC-V G-stage paging mode"), - } - } - - fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - match levels { - 3 => Ok(NestedPagingConfig::new(root_paddr, 3, 41, 8)), - 4 => Ok(NestedPagingConfig::new(root_paddr, 4, 50, 9)), - _ => ax_err!(InvalidInput, "unsupported RISC-V G-stage levels"), - } - } - - fn new_nested_page_table(levels: usize) -> AxResult { - npt::NestedPageTable::new(levels) - } - - fn new_vcpu_create_state( - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - Ok(()) - } - - fn build_vcpu_create_config( - _state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig> { - let (vcpu_id, _phys_cpu_id, dtb_addr, _firmware_boot) = ctx.into_parts(); - Ok(RiscvVcpuCreateConfig { - hart_id: vcpu_id, - dtb_addr: dtb_addr.unwrap_or_default().as_usize(), - }) - } - - fn build_vcpu_setup_config( - _config: &crate::config::AxVMConfig, - _memory_regions: &[crate::vm::VMMemoryRegion], - ) -> AxResult<::SetupConfig> { - Ok(()) - } - fn register_platform_irq_injector() { register_platform_irq_injector(); } diff --git a/virtualization/axvm/src/arch/riscv64/vm.rs b/virtualization/axvm/src/arch/riscv64/vm.rs new file mode 100644 index 0000000000..c525ff12db --- /dev/null +++ b/virtualization/axvm/src/arch/riscv64/vm.rs @@ -0,0 +1,110 @@ +//! RISC-V VM resource creation and initialization. + +use ax_errno::{AxResult, ax_err}; +use axvm_types::{NestedPagingConfig, VmArchVcpuOps}; +use riscv_vcpu::RiscvVcpuCreateConfig; + +use super::{Riscv64Arch, irq, npt}; +use crate::{ + config::AxVMConfig, + vm::{ + AxVM, AxVMResources, + prepare::{ + PreparedVm, VmInitRequest, + address_space::{guest_owned_regions, map_guest_address_space}, + complete_vm_init, default_device_factories, + devices::PreparedDevices, + validate_guest_dtb, + vcpus::{PreparedVcpus, vcpu_placements}, + }, + }, +}; + +impl Riscv64Arch { + pub(crate) fn create_vm_resources(config: AxVMConfig) -> AxResult { + let placements = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); + let levels = guest_page_table_levels(&placements)?; + let page_table = npt::NestedPageTable::new(levels)?; + AxVMResources::from_page_table(config, page_table, |root_paddr| { + nested_paging_config(root_paddr, levels) + }) + } + + pub(crate) fn init_vm(vm: &AxVM, request: VmInitRequest<'_>) -> AxResult { + match request { + VmInitRequest::Default => { + let mut factories = default_device_factories()?; + let mode = vm.interrupt_mode(); + let emulated_devices = vm.with_config(|config| config.emu_devices().clone()); + let interrupt_fabric = irq::configure(&mut factories, mode, &emulated_devices)?; + init_vm_with(vm, &factories, interrupt_fabric) + } + VmInitRequest::Provided { + factories, + interrupt_fabric, + } => init_vm_with(vm, factories, interrupt_fabric), + } + } +} + +fn init_vm_with( + vm: &AxVM, + factories: &axdevice::DeviceFactoryRegistry, + interrupt_fabric: crate::InterruptFabric, +) -> AxResult { + complete_vm_init(vm, interrupt_fabric, |resources, interrupt_fabric| { + let placements = vcpu_placements(resources); + let dtb_addr = resources + .config() + .image_config() + .dtb_load_gpa + .unwrap_or_default(); + let vcpus = PreparedVcpus::create(vm.id(), &placements, |placement| { + Ok(RiscvVcpuCreateConfig { + hart_id: placement.id, + dtb_addr: dtb_addr.as_usize(), + }) + })?; + let mut devices = PreparedDevices::build_common(resources, factories, interrupt_fabric)?; + devices.register_special_devices(vm)?; + validate_guest_dtb(resources)?; + + let owned_regions = guest_owned_regions(resources); + map_guest_address_space(vm, resources, devices.devices(), &owned_regions)?; + vcpus.setup(resources, build_vcpu_setup_config)?; + + Ok(PreparedVm::new(vcpus, devices)) + }) +} + +fn build_vcpu_setup_config( + _config: &AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], +) -> AxResult<::SetupConfig> { + Ok(()) +} + +fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { + let mut levels = riscv_vcpu::max_guest_page_table_levels(); + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + levels = levels.min( + crate::percpu::cpu_max_guest_page_table_levels(cpu_id) + .unwrap_or_else(riscv_vcpu::max_guest_page_table_levels), + ); + } + match levels { + 3 | 4 => Ok(levels), + _ => ax_err!(Unsupported, "no supported RISC-V G-stage paging mode"), + } +} + +fn nested_paging_config( + root_paddr: ax_memory_addr::PhysAddr, + levels: usize, +) -> AxResult { + match levels { + 3 => Ok(NestedPagingConfig::new(root_paddr, 3, 41, 8)), + 4 => Ok(NestedPagingConfig::new(root_paddr, 4, 50, 9)), + _ => ax_err!(InvalidInput, "unsupported RISC-V G-stage levels"), + } +} diff --git a/virtualization/axvm/src/arch/x86_64/capabilities.rs b/virtualization/axvm/src/arch/x86_64/capabilities.rs index 3584487dff..685d12a588 100644 --- a/virtualization/axvm/src/arch/x86_64/capabilities.rs +++ b/virtualization/axvm/src/arch/x86_64/capabilities.rs @@ -1,67 +1,7 @@ //! x86_64 implementations of AxVM platform capability hooks. -use alloc::{format, sync::Arc}; - -use ax_errno::{AxResult, ax_err_type}; -use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; - use super::X86_64Arch; -use crate::architecture::{ - AddressSpacePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, -}; - -impl DevicePlatform for X86_64Arch { - fn register_devices( - _vm: &crate::AxVM, - config: &crate::config::AxVMConfig, - devices: &mut axdevice::AxVmDevices, - ) -> AxResult { - for port in config.pass_through_ports() { - let passthrough = Arc::new(super::port::HostPortPassthrough::new( - port.base, - port.length, - )?); - let range = passthrough.address_range(); - debug!( - "PT port region: [{:#x}~{:#x}]", - range.start.number(), - range.end.number(), - ); - devices - .register(PortDeviceAdapter::from_arc(passthrough)) - .map_err(|err| ax_err_type!(InvalidInput, format!("register PT port: {err:?}")))?; - } - for config in config.emu_devices() { - super::register_arch_device(config, devices)?; - } - Ok(()) - } -} - -impl AddressSpacePlatform for X86_64Arch { - #[cfg(feature = "vmx")] - fn append_owned_regions(regions: &mut alloc::vec::Vec) { - regions.push(crate::layout::GuestOwnedRegion::new( - x86_vcpu::X86_APIC_ACCESS_GPA, - ax_memory_addr::PAGE_SIZE_4K, - crate::layout::VmRegionKind::Reserved, - )); - } - - #[cfg(feature = "vmx")] - fn map_address_space( - address_space: &mut axaddrspace::AddrSpace, - ) -> AxResult { - address_space.map_linear( - axvm_types::GuestPhysAddr::from(x86_vcpu::X86_APIC_ACCESS_GPA), - super::x86_apic_access_page_addr(), - ax_memory_addr::PAGE_SIZE_4K, - axvm_types::MappingFlags::DEVICE - | axvm_types::MappingFlags::READ - | axvm_types::MappingFlags::WRITE, - ) - } -} +use crate::architecture::{GuestBootPlatform, HostTimePlatform}; impl HostTimePlatform for X86_64Arch {} diff --git a/virtualization/axvm/src/arch/x86_64/mod.rs b/virtualization/axvm/src/arch/x86_64/mod.rs index 9d8575efbb..aa5bbe4bf5 100644 --- a/virtualization/axvm/src/arch/x86_64/mod.rs +++ b/virtualization/axvm/src/arch/x86_64/mod.rs @@ -22,10 +22,7 @@ use x86_vlapic::{ X86VlapicResult, X86VmId, }; -use super::{ - ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, -}; +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; use crate::{ StopReason, host::{HostMemory, default_host}, @@ -43,6 +40,7 @@ mod npt; pub(crate) mod port; #[path = "../../architecture/sysreg.rs"] mod sysreg; +mod vm; use exit::{DeferredRunWork, IoReadExit, IoWriteExit, NestedPageFaultExit}; use sysreg::{SysRegReadExit, SysRegWriteExit}; @@ -53,12 +51,9 @@ const RFLAGS_INTERRUPT_FLAG: u64 = 1 << 9; pub(crate) struct X86_64Arch; -pub(crate) struct X86VcpuCreateState; - impl ArchOps for X86_64Arch { type VCpu = AxvmX86Vcpu; type PerCpu = AxvmX86PerCpu; - type VcpuCreateState = X86VcpuCreateState; type DeferredRunWork = DeferredRunWork; type NestedPageTable = npt::NestedPageTable; @@ -66,49 +61,6 @@ impl ArchOps for X86_64Arch { x86_vcpu::has_hardware_support() } - fn new_vcpu_create_state( - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - Ok(X86VcpuCreateState) - } - - fn build_vcpu_create_config( - _state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig> { - let _ = ctx.into_parts(); - Ok(X86VCpuCreateConfig) - } - - fn build_vcpu_setup_config( - vm_config: &crate::config::AxVMConfig, - memory_regions: &[crate::vm::VMMemoryRegion], - ) -> AxResult<::SetupConfig> { - let mut setup_config = X86VCpuSetupConfig { - emulate_com1: vm_config - .emu_devices() - .iter() - .any(|device| device.emu_type == EmulatedDeviceType::Console), - guest_memory_regions: memory_regions - .iter() - .map(|region| x86_vcpu::X86GuestMemoryRegion { - gpa: X86GuestPhysAddr::from_usize(region.gpa.as_usize()), - hva: X86HostVirtAddr::from_usize(region.hva.as_usize()), - size: region.size(), - }) - .collect(), - ..Default::default() - }; - for port in vm_config.pass_through_ports() { - x86_result(setup_config.add_passthrough_port_range(port.base, port.length))?; - } - Ok(setup_config) - } - - fn new_nested_page_table(levels: usize) -> AxResult { - npt::NestedPageTable::new(levels) - } - fn before_first_run(vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { irq::enable_ioapic_irq_forwarding(vm, vcpu); } diff --git a/virtualization/axvm/src/arch/x86_64/vm.rs b/virtualization/axvm/src/arch/x86_64/vm.rs new file mode 100644 index 0000000000..f852a8aa2b --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/vm.rs @@ -0,0 +1,173 @@ +//! x86_64 VM resource creation and initialization. + +use alloc::sync::Arc; + +use ax_errno::{AxResult, ax_err_type}; +#[cfg(feature = "vmx")] +use ax_memory_addr::PAGE_SIZE_4K; +use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; +#[cfg(feature = "vmx")] +use axvm_types::MappingFlags; +use axvm_types::{EmulatedDeviceType, NestedPagingConfig, VmArchVcpuOps}; +use x86_vcpu::{ + X86GuestMemoryRegion, X86GuestPhysAddr, X86HostVirtAddr, X86VCpuCreateConfig, + X86VCpuSetupConfig, +}; + +#[cfg(feature = "vmx")] +use super::x86_apic_access_page_addr; +use super::{X86_64Arch, npt, x86_result}; +use crate::{ + config::AxVMConfig, + layout::GuestOwnedRegion, + vm::{ + AxVM, AxVMResources, + prepare::{ + PreparedVm, VmInitRequest, + address_space::{guest_owned_regions, map_guest_address_space}, + complete_vm_init, default_device_factories, + devices::PreparedDevices, + validate_guest_dtb, + vcpus::{PreparedVcpus, vcpu_placements}, + }, + }, +}; + +impl X86_64Arch { + pub(crate) fn create_vm_resources(config: AxVMConfig) -> AxResult { + let placements = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); + let levels = guest_page_table_levels(&placements); + let page_table = npt::NestedPageTable::new(levels)?; + AxVMResources::from_page_table(config, page_table, |root_paddr| { + let gpa_bits = match levels { + 3 => 39, + 4 => 48, + _ => { + return ax_errno::ax_err!( + InvalidInput, + "unsupported x86 nested page-table levels" + ); + } + }; + Ok(NestedPagingConfig::new(root_paddr, levels, gpa_bits, 0)) + }) + } + + pub(crate) fn init_vm(vm: &AxVM, request: VmInitRequest<'_>) -> AxResult { + match request { + VmInitRequest::Default => { + let factories = default_device_factories()?; + let interrupt_fabric = crate::InterruptFabric::new(vm.interrupt_mode()); + init_vm_with(vm, &factories, interrupt_fabric) + } + VmInitRequest::Provided { + factories, + interrupt_fabric, + } => init_vm_with(vm, factories, interrupt_fabric), + } + } +} + +fn init_vm_with( + vm: &AxVM, + factories: &axdevice::DeviceFactoryRegistry, + interrupt_fabric: crate::InterruptFabric, +) -> AxResult { + complete_vm_init(vm, interrupt_fabric, |resources, interrupt_fabric| { + let placements = vcpu_placements(resources); + let vcpus = PreparedVcpus::create(vm.id(), &placements, |_| Ok(X86VCpuCreateConfig))?; + let mut devices = PreparedDevices::build_common(resources, factories, interrupt_fabric)?; + register_arch_devices(resources.config(), &mut devices.devices)?; + devices.register_special_devices(vm)?; + validate_guest_dtb(resources)?; + + let mut owned_regions = guest_owned_regions(resources); + append_arch_owned_regions(&mut owned_regions); + map_guest_address_space(vm, resources, devices.devices(), &owned_regions)?; + map_arch_address_space(resources)?; + vcpus.setup(resources, build_vcpu_setup_config)?; + + Ok(PreparedVm::new(vcpus, devices)) + }) +} + +fn build_vcpu_setup_config( + config: &AxVMConfig, + memory_regions: &[crate::vm::VMMemoryRegion], +) -> AxResult<::SetupConfig> { + let mut setup_config = X86VCpuSetupConfig { + emulate_com1: config + .emu_devices() + .iter() + .any(|device| device.emu_type == EmulatedDeviceType::Console), + guest_memory_regions: memory_regions + .iter() + .map(|region| X86GuestMemoryRegion { + gpa: X86GuestPhysAddr::from_usize(region.gpa.as_usize()), + hva: X86HostVirtAddr::from_usize(region.hva.as_usize()), + size: region.size(), + }) + .collect(), + ..Default::default() + }; + for port in config.pass_through_ports() { + x86_result(setup_config.add_passthrough_port_range(port.base, port.length))?; + } + Ok(setup_config) +} + +fn register_arch_devices(config: &AxVMConfig, devices: &mut axdevice::AxVmDevices) -> AxResult { + for port in config.pass_through_ports() { + let passthrough = Arc::new(super::port::HostPortPassthrough::new( + port.base, + port.length, + )?); + let range = passthrough.address_range(); + debug!( + "PT port region: [{:#x}~{:#x}]", + range.start.number(), + range.end.number(), + ); + devices + .register(PortDeviceAdapter::from_arc(passthrough)) + .map_err(|err| { + ax_err_type!(InvalidInput, alloc::format!("register PT port: {err:?}")) + })?; + } + for device_config in config.emu_devices() { + super::register_arch_device(device_config, devices)?; + } + Ok(()) +} + +fn append_arch_owned_regions(regions: &mut alloc::vec::Vec) { + #[cfg(feature = "vmx")] + regions.push(GuestOwnedRegion::new( + x86_vcpu::X86_APIC_ACCESS_GPA, + PAGE_SIZE_4K, + crate::layout::VmRegionKind::Reserved, + )); + #[cfg(not(feature = "vmx"))] + let _ = regions; +} + +fn map_arch_address_space(resources: &mut AxVMResources) -> AxResult { + #[cfg(feature = "vmx")] + resources.address_space.map_linear( + axvm_types::GuestPhysAddr::from(x86_vcpu::X86_APIC_ACCESS_GPA), + x86_apic_access_page_addr(), + PAGE_SIZE_4K, + MappingFlags::DEVICE | MappingFlags::READ | MappingFlags::WRITE, + )?; + #[cfg(not(feature = "vmx"))] + let _ = resources; + Ok(()) +} + +fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> usize { + let mut levels = 4; + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + levels = levels.min(crate::percpu::cpu_max_guest_page_table_levels(cpu_id).unwrap_or(4)); + } + levels +} diff --git a/virtualization/axvm/src/architecture/capabilities.rs b/virtualization/axvm/src/architecture/capabilities.rs index e08ac927e7..2cd187c217 100644 --- a/virtualization/axvm/src/architecture/capabilities.rs +++ b/virtualization/axvm/src/architecture/capabilities.rs @@ -2,8 +2,6 @@ use ax_errno::AxResult; -use super::ArchOps; - /// Guest firmware preparation performed before common VM memory loading. pub(crate) trait GuestBootPlatform { fn init_guest_boot_resources() {} @@ -54,36 +52,6 @@ pub(crate) trait BootImagePlatform { } } -/// Architecture-owned interrupt fabric and device registration hooks. -pub(crate) trait DevicePlatform { - fn configure_interrupt_fabric( - _factories: &mut axdevice::DeviceFactoryRegistry, - mode: axvm_types::VMInterruptMode, - _configs: &[axvm_types::EmulatedDeviceConfig], - ) -> AxResult { - Ok(crate::InterruptFabric::new(mode)) - } - - fn register_devices( - _vm: &crate::AxVM, - _config: &crate::config::AxVMConfig, - _devices: &mut axdevice::AxVmDevices, - ) -> AxResult { - Ok(()) - } -} - -/// Architecture-owned guest address-space additions. -pub(crate) trait AddressSpacePlatform: ArchOps { - fn append_owned_regions(_regions: &mut alloc::vec::Vec) {} - - fn map_address_space( - _address_space: &mut axaddrspace::AddrSpace, - ) -> AxResult { - Ok(()) - } -} - /// Architecture-specific host timer policy used by the ArceOS adapter. pub(crate) trait HostTimePlatform { fn set_oneshot_timer(deadline_ns: u64) { diff --git a/virtualization/axvm/src/architecture/mod.rs b/virtualization/axvm/src/architecture/mod.rs index 926d40051d..7d10f101d2 100644 --- a/virtualization/axvm/src/architecture/mod.rs +++ b/virtualization/axvm/src/architecture/mod.rs @@ -5,11 +5,7 @@ mod exit; pub(crate) mod ops; mod types; -pub(crate) use capabilities::{ - AddressSpacePlatform, BootImagePlatform, DevicePlatform, GuestBootPlatform, HostTimePlatform, -}; +pub(crate) use capabilities::{BootImagePlatform, GuestBootPlatform, HostTimePlatform}; pub(crate) use exit::{handle_hypercall, handle_mmio_read, handle_mmio_write}; pub(crate) use ops::ArchOps; -pub(crate) use types::{ - BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, VcpuRunAction, -}; +pub(crate) use types::{BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; diff --git a/virtualization/axvm/src/architecture/ops.rs b/virtualization/axvm/src/architecture/ops.rs index 4c2341bc6a..144d9522b7 100644 --- a/virtualization/axvm/src/architecture/ops.rs +++ b/virtualization/axvm/src/architecture/ops.rs @@ -3,67 +3,22 @@ use alloc::{format, vec::Vec}; use ax_errno::{AxResult, ax_err}; -use ax_memory_addr::{PhysAddr, VirtAddr}; +use ax_memory_addr::VirtAddr; use axaddrspace::NestedPageTableOps; -use axvm_types::{NestedPagingConfig, VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState}; +use axvm_types::{VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState}; -use super::{BoundVcpuExit, VcpuCreateContext, VcpuRunAction}; +use super::{BoundVcpuExit, VcpuRunAction}; pub(crate) trait ArchOps { type VCpu: VmArchVcpuOps; type PerCpu: VmArchPerCpuOps; - type VcpuCreateState; type DeferredRunWork; type NestedPageTable: NestedPageTableOps; fn has_hardware_support() -> bool; - fn max_guest_page_table_levels() -> usize { - 4 - } - - fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { - let mut levels = Self::max_guest_page_table_levels(); - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - levels = levels.min( - crate::percpu::cpu_max_guest_page_table_levels(cpu_id) - .unwrap_or_else(Self::max_guest_page_table_levels), - ); - } - Ok(levels) - } - - fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - let gpa_bits = match levels { - 3 => 39, - 4 => 48, - _ => return ax_errno::ax_err!(InvalidInput, "unsupported nested page-table levels"), - }; - Ok(NestedPagingConfig::new(root_paddr, levels, gpa_bits, 0)) - } - - fn new_nested_page_table(levels: usize) -> AxResult; - fn clean_dcache_range(_addr: VirtAddr, _size: usize) {} - fn new_vcpu_create_state( - vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult; - - fn build_vcpu_create_config( - state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig>; - - fn build_vcpu_setup_config( - config: &crate::config::AxVMConfig, - memory_regions: &[crate::vm::VMMemoryRegion], - ) -> AxResult<::SetupConfig>; - fn register_platform_irq_injector() {} fn vcpu_affinities( diff --git a/virtualization/axvm/src/architecture/types.rs b/virtualization/axvm/src/architecture/types.rs index 10fbd71030..2bdfaf4141 100644 --- a/virtualization/axvm/src/architecture/types.rs +++ b/virtualization/axvm/src/architecture/types.rs @@ -43,21 +43,3 @@ pub(crate) struct HypercallExit { pub(crate) nr: u64, pub(crate) args: [u64; 6], } - -pub(crate) struct VcpuCreateContext { - pub(crate) vcpu_id: usize, - pub(crate) phys_cpu_id: usize, - pub(crate) dtb_addr: Option, - pub(crate) firmware_boot: bool, -} - -impl VcpuCreateContext { - pub(crate) fn into_parts(self) -> (usize, usize, Option, bool) { - ( - self.vcpu_id, - self.phys_cpu_id, - self.dtb_addr, - self.firmware_boot, - ) - } -} diff --git a/virtualization/axvm/src/config.rs b/virtualization/axvm/src/config.rs index 264a1ebac2..46aca8d0a2 100644 --- a/virtualization/axvm/src/config.rs +++ b/virtualization/axvm/src/config.rs @@ -240,15 +240,6 @@ impl AxVMConfig { self.boot_policy } - pub(crate) fn uses_firmware_boot(&self) -> bool { - matches!( - self.boot_policy, - GuestBootPolicy::AdjustKernelForBootProtocol { - protocol: VMBootProtocol::Uefi, - } - ) - } - /// Sets the policy used to adjust runtime boot image addresses. pub fn set_boot_policy(&mut self, boot_policy: GuestBootPolicy) { self.boot_policy = boot_policy; @@ -435,16 +426,4 @@ mod tests { assert_eq!(regions[1].size, 0x10000); assert_eq!(regions[1].map_type, VmMemMappingType::MapReserved); } - - #[test] - fn firmware_boot_is_derived_from_the_typed_boot_policy() { - let mut config = AxVMConfig::default_for_test(1, "linux"); - assert!(!config.uses_firmware_boot()); - - config.set_boot_policy(GuestBootPolicy::AdjustKernelForBootProtocol { - protocol: VMBootProtocol::Uefi, - }); - - assert!(config.uses_firmware_boot()); - } } diff --git a/virtualization/axvm/src/lifecycle/machine.rs b/virtualization/axvm/src/lifecycle/machine.rs index cf79b8be40..79e5819a79 100644 --- a/virtualization/axvm/src/lifecycle/machine.rs +++ b/virtualization/axvm/src/lifecycle/machine.rs @@ -1,13 +1,8 @@ -use alloc::{ - boxed::Box, - string::{String, ToString}, -}; +use alloc::string::{String, ToString}; use super::{StopReason, VmLifecycleError, VmLifecycleResult, VmStatus}; -use crate::config::AxVMConfig; pub enum Machine { - Uninit(Box), Ready(R), Running { resources: R, @@ -40,7 +35,6 @@ pub enum Machine { impl Machine { pub fn status(&self) -> VmStatus { match self { - Machine::Uninit(_) => VmStatus::Uninit, Machine::Ready(_) => VmStatus::Ready, Machine::Running { .. } => VmStatus::Running, Machine::Pausing { .. } => VmStatus::Pausing, @@ -80,41 +74,6 @@ impl Machine { } } - pub fn config_mut(&mut self) -> Option<&mut AxVMConfig> { - match self { - Machine::Uninit(config) => Some(config.as_mut()), - _ => None, - } - } - - pub fn prepare_with(&mut self, f: F) -> VmLifecycleResult - where - F: FnOnce(AxVMConfig) -> VmLifecycleResult, - { - let old = core::mem::replace(self, Machine::Switching); - match old { - Machine::Uninit(config) => match f(*config) { - Ok(resources) => { - *self = Machine::Ready(resources); - Ok(()) - } - Err(err) => { - *self = Machine::Failed(err.to_string()); - Err(err) - } - }, - other => { - let from = other.status(); - *self = other; - Err(VmLifecycleError::invalid_transition( - from, - VmStatus::Ready, - "prepare", - )) - } - } - } - pub fn runtime(&self) -> Option<&H> { match self { Machine::Running { runtime, .. } @@ -579,7 +538,7 @@ impl Machine { *self = Machine::Destroyed; Ok(()) } - Machine::Uninit(_) | Machine::Failed(_) | Machine::Switching | Machine::Destroying => { + Machine::Failed(_) | Machine::Switching | Machine::Destroying => { f(None)?; *self = Machine::Destroyed; Ok(()) @@ -592,16 +551,9 @@ impl Machine { mod tests { use super::*; - fn config() -> AxVMConfig { - AxVMConfig::default_for_test(1, "lifecycle-test") - } - #[test] - fn lifecycle_allows_prepare_start_pause_resume_stop_destroy() { - let mut machine = Machine::Uninit(Box::new(config())); - assert_eq!(machine.status(), VmStatus::Uninit); - - machine.prepare_with(|_| Ok(7usize)).unwrap(); + fn lifecycle_allows_start_pause_resume_stop_destroy_from_ready() { + let mut machine = Machine::Ready(7usize); assert_eq!(machine.status(), VmStatus::Ready); machine @@ -639,19 +591,7 @@ mod tests { #[test] fn lifecycle_rejects_invalid_transitions_without_changing_state() { - let mut machine = Machine::::Uninit(Box::new(config())); - let err = machine.start_with(|_| Ok(())).unwrap_err(); - assert!(matches!( - err, - VmLifecycleError::InvalidTransition { - from: VmStatus::Uninit, - to: VmStatus::Running, - op: "start" - } - )); - assert_eq!(machine.status(), VmStatus::Uninit); - - machine.prepare_with(|_| Ok(1)).unwrap(); + let mut machine = Machine::::Ready(1); let err = machine.resume().unwrap_err(); assert!(matches!( err, diff --git a/virtualization/axvm/src/lifecycle/status.rs b/virtualization/axvm/src/lifecycle/status.rs index ab7fd12c15..9b987ee1ca 100644 --- a/virtualization/axvm/src/lifecycle/status.rs +++ b/virtualization/axvm/src/lifecycle/status.rs @@ -11,7 +11,6 @@ pub enum StopReason { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum VmStatus { - Uninit, Ready, Running, Pausing, @@ -26,7 +25,6 @@ pub enum VmStatus { impl VmStatus { pub const fn as_str(self) -> &'static str { match self { - VmStatus::Uninit => "uninit", VmStatus::Ready => "ready", VmStatus::Running => "running", VmStatus::Pausing => "pausing", @@ -45,7 +43,6 @@ impl VmStatus { pub const fn as_str_with_icon(self) -> &'static str { match self { - VmStatus::Uninit => "[..] uninit", VmStatus::Ready => "[OK] ready", VmStatus::Running => "[RUN] running", VmStatus::Pausing => "[..] pausing", diff --git a/virtualization/axvm/src/vm/mod.rs b/virtualization/axvm/src/vm/mod.rs index 01e427e6ef..c193bdc94b 100644 --- a/virtualization/axvm/src/vm/mod.rs +++ b/virtualization/axvm/src/vm/mod.rs @@ -44,7 +44,7 @@ use crate::{ pub(crate) mod boot; pub(crate) mod memory; -mod prepare; +pub(crate) mod prepare; pub use memory::PreparedMemoryLayout; const VM_ASPACE_BASE: usize = 0x0; @@ -113,7 +113,7 @@ fn write_guest_bytes_to_chunks(chunks: &mut [&mut [u8]], data: &[u8]) -> AxResul pub(crate) struct AxVMResources { // Todo: use more efficient lock. - address_space: AddrSpace, + pub(crate) address_space: AddrSpace, nested_paging: NestedPagingConfig, memory_regions: Vec, config: AxVMConfig, @@ -263,20 +263,17 @@ impl VmRuntimeHandle { } impl AxVMResources { - fn new(config: AxVMConfig) -> AxResult { - let vcpu_mappings = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); - let page_table_levels = crate::arch::guest_page_table_levels(&vcpu_mappings)?; - let page_table = crate::arch::new_nested_page_table(page_table_levels)?; + pub(crate) fn from_page_table( + config: AxVMConfig, + page_table: ArchNestedPageTable, + build_nested_paging: impl FnOnce(HostPhysAddr) -> AxResult, + ) -> AxResult { let address_space = AddrSpace::new_empty( page_table, GuestPhysAddr::from(VM_ASPACE_BASE), VM_ASPACE_SIZE, )?; - let nested_paging = crate::arch::nested_paging_config( - address_space.page_table_root(), - page_table_levels, - &vcpu_mappings, - )?; + let nested_paging = build_nested_paging(address_space.page_table_root())?; Ok(Self { address_space, nested_paging, @@ -291,6 +288,10 @@ impl AxVMResources { }) } + pub(crate) const fn config(&self) -> &AxVMConfig { + &self.config + } + fn vcpu_list(&self) -> AxResult<&[AxVCpuRef]> { self.vcpu_list .as_deref() @@ -392,16 +393,22 @@ pub struct AxVM { } impl AxVM { - /// Creates a new VM with the given configuration. - /// Returns an error if the configuration is invalid. - /// The VM is not started until `start` is called. + /// Creates a ready VM with eagerly initialized architecture resources. + /// + /// The VM is not started until [`Self::start`] is called. + /// + /// # Errors + /// + /// Returns an error if nested paging is unsupported for the selected host + /// CPUs or if the initial stage-2 address space cannot be allocated. pub fn new(config: AxVMConfig) -> AxResult { let id = config.id(); let name = config.name(); + let resources = crate::arch::CurrentArch::create_vm_resources(config)?; let result = Arc::new(Self { id, name, - machine: Mutex::new(Machine::Uninit(Box::new(config))), + machine: Mutex::new(Machine::Ready(resources)), pending_fw_cfg: Mutex::new(None), }); @@ -428,33 +435,14 @@ impl AxVM { /// Returns the configured VM interrupt mode. pub fn interrupt_mode(&self) -> VMInterruptMode { - let _ = self.ensure_resources_ready(); self.with_resources(|resources| Ok(resources.config.interrupt_mode())) .unwrap_or(VMInterruptMode::NoIrq) } - fn ensure_resources_ready(&self) -> AxResult { - let mut machine = self.machine.lock(); - if !matches!(machine.status(), VmStatus::Uninit) { - return Ok(()); - } - machine - .prepare_with(|config| { - AxVMResources::new(config).map_err(|err| { - VmLifecycleError::MissingResource(match err { - AxError::NoMemory => "VM address space memory", - _ => "VM address space", - }) - }) - }) - .map_err(VmLifecycleError::into_ax_error) - } - fn with_resources(&self, f: F) -> AxResult where F: FnOnce(&AxVMResources) -> AxResult, { - self.ensure_resources_ready()?; let machine = self.machine.lock(); let resources = machine .resources() @@ -466,7 +454,6 @@ impl AxVM { where F: FnOnce(&mut AxVMResources) -> AxResult, { - self.ensure_resources_ready()?; let mut machine = self.machine.lock(); let resources = machine .resources_mut() @@ -533,20 +520,6 @@ impl AxVM { F: FnOnce(&mut AxVMConfig) -> R, { let mut machine = self.machine.lock(); - if matches!(machine.status(), VmStatus::Uninit) { - let old = core::mem::replace(&mut *machine, Machine::Switching); - if let Machine::Uninit(config) = old { - match AxVMResources::new(*config) { - Ok(resources) => *machine = Machine::Ready(resources), - Err(err) => { - *machine = Machine::Failed(format!("prepare config resources: {err:?}")); - panic!("VM resources are not available: {err:?}"); - } - } - } else { - *machine = old; - } - } let resources = machine .resources_mut() .expect("VM resources are not available for config access"); @@ -590,7 +563,6 @@ impl AxVM { /// Starts the VM by transitioning to Running state. pub fn start(self: &Arc) -> AxResult { - self.ensure_resources_ready()?; if self.status() == VmStatus::Stopped { if let Some(runtime) = self.take_stopped_runtime() { runtime.join_all_vcpu_tasks(self.id()); @@ -1250,7 +1222,7 @@ impl AxVM { VmStatus::Running | VmStatus::Paused | VmStatus::Stopping => { self.stop_and_join_runtime(StopReason::Forced)?; } - VmStatus::Ready | VmStatus::Stopped | VmStatus::Uninit | VmStatus::Failed => { + VmStatus::Ready | VmStatus::Stopped | VmStatus::Failed => { if let Some(runtime) = self.take_stopped_runtime() { runtime.join_all_vcpu_tasks(vm_id); } diff --git a/virtualization/axvm/src/vm/prepare.rs b/virtualization/axvm/src/vm/prepare.rs index 2b236bd253..4f57cd84ba 100644 --- a/virtualization/axvm/src/vm/prepare.rs +++ b/virtualization/axvm/src/vm/prepare.rs @@ -1,8 +1,8 @@ -//! VM preparation orchestration. +//! Architecture-neutral mechanics used by architecture-owned VM initialization. -mod address_space; -mod devices; -mod vcpus; +pub(crate) mod address_space; +pub(crate) mod devices; +pub(crate) mod vcpus; use alloc::{format, sync::Arc}; @@ -10,24 +10,32 @@ use ax_errno::{AxResult, ax_err, ax_err_type}; use axdevice::{DeviceFactoryRegistry, register_builtin_factories}; use self::{devices::PreparedDevices, vcpus::PreparedVcpus}; -use super::AxVM; +use super::{AxVM, AxVMResources}; use crate::irq::InterruptFabric; +pub(crate) enum VmInitRequest<'a> { + Default, + Provided { + factories: &'a DeviceFactoryRegistry, + interrupt_fabric: InterruptFabric, + }, +} + +pub(crate) struct PreparedVm { + vcpus: PreparedVcpus, + devices: PreparedDevices, +} + +impl PreparedVm { + pub(crate) const fn new(vcpus: PreparedVcpus, devices: PreparedDevices) -> Self { + Self { vcpus, devices } + } +} + impl AxVM { /// Sets up the VM before booting. pub fn prepare(&self) -> AxResult { - self.ensure_resources_ready()?; - let mut factories = DeviceFactoryRegistry::new(); - register_builtin_factories(&mut factories)?; - let interrupt_mode = self.interrupt_mode(); - let emulated_devices = self.with_config(|config| config.emu_devices().clone()); - let interrupt_fabric = crate::arch::configure_interrupt_fabric( - &mut factories, - interrupt_mode, - &emulated_devices, - )?; - - self.prepare_with_factories(&factories, interrupt_fabric) + crate::arch::CurrentArch::init_vm(self, VmInitRequest::Default) } /// Sets up the VM with explicit device factories and an interrupt fabric. @@ -36,52 +44,73 @@ impl AxVM { factories: &DeviceFactoryRegistry, interrupt_fabric: InterruptFabric, ) -> AxResult { - self.ensure_resources_ready()?; - let mut machine = self.machine.lock(); - if !matches!( - machine.status(), - crate::lifecycle::VmStatus::Ready | crate::lifecycle::VmStatus::Stopped - ) { - return ax_err!( - BadState, - format!( - "VM[{}] cannot prepare from {:?}", - self.id(), - machine.status() - ) - ); - } - let resources = machine - .resources_mut() - .ok_or_else(|| ax_err_type!(BadState, "VM resources are not available for prepare"))?; - if resources.vcpu_list.is_some() - || resources.devices.is_some() - || resources.interrupt_fabric.is_some() - { - resources.reset_transient_resources()?; - } - interrupt_fabric.validate_mode(resources.config.interrupt_mode())?; + crate::arch::CurrentArch::init_vm( + self, + VmInitRequest::Provided { + factories, + interrupt_fabric, + }, + ) + } +} - let dtb_addr = resources.config.image_config().dtb_load_gpa; - let vcpus = PreparedVcpus::create(self.id(), resources, dtb_addr)?; - let devices = PreparedDevices::build(self, resources, factories, &interrupt_fabric)?; +pub(crate) fn default_device_factories() -> AxResult { + let mut factories = DeviceFactoryRegistry::new(); + register_builtin_factories(&mut factories)?; + Ok(factories) +} - if dtb_addr.is_some() && resources.boot_description.device_tree().is_none() { - return ax_err!( - InvalidInput, - "DTB load GPA is configured but no guest device tree bytes are registered" - ); - } +pub(crate) fn complete_vm_init( + vm: &AxVM, + interrupt_fabric: InterruptFabric, + initialize: impl FnOnce(&mut AxVMResources, &InterruptFabric) -> AxResult, +) -> AxResult { + let mut machine = vm.machine.lock(); + if !matches!( + machine.status(), + crate::lifecycle::VmStatus::Ready | crate::lifecycle::VmStatus::Stopped + ) { + return ax_err!( + BadState, + format!("VM[{}] cannot prepare from {:?}", vm.id(), machine.status()) + ); + } + let resources = machine + .resources_mut() + .ok_or_else(|| ax_err_type!(BadState, "VM resources are not available for prepare"))?; + resources.reset_transient_resources()?; + interrupt_fabric.validate_mode(resources.config.interrupt_mode())?; - address_space::map_guest_address_space(self, resources, devices.devices())?; - vcpus.setup(resources)?; + let prepared = match initialize(resources, &interrupt_fabric) { + Ok(prepared) => prepared, + Err(err) => { + if let Err(reset_err) = resources.reset_transient_resources() { + warn!( + "VM[{}] failed to reset transient resources after initialization error: \ + {reset_err:?}", + vm.id() + ); + } + return Err(err); + } + }; + resources.phys_cpu_ls = resources.config.phys_cpu_ls.clone(); + resources.vcpu_list = Some(prepared.vcpus.into_boxed_slice()); + resources.devices = Some(Arc::new(prepared.devices.into_inner())); + resources.interrupt_fabric = Some(interrupt_fabric); - resources.phys_cpu_ls = resources.config.phys_cpu_ls.clone(); - resources.vcpu_list = Some(vcpus.into_boxed_slice()); - resources.devices = Some(Arc::new(devices.into_inner())); - resources.interrupt_fabric = Some(interrupt_fabric); + info!("VM setup: id={}", vm.id()); + Ok(()) +} - info!("VM setup: id={}", self.id()); - Ok(()) +pub(crate) fn validate_guest_dtb(resources: &AxVMResources) -> AxResult { + if resources.config.image_config().dtb_load_gpa.is_some() + && resources.boot_description.device_tree().is_none() + { + return ax_err!( + InvalidInput, + "DTB load GPA is configured but no guest device tree bytes are registered" + ); } + Ok(()) } diff --git a/virtualization/axvm/src/vm/prepare/address_space.rs b/virtualization/axvm/src/vm/prepare/address_space.rs index 2cc356dcd6..c829feeb37 100644 --- a/virtualization/axvm/src/vm/prepare/address_space.rs +++ b/virtualization/axvm/src/vm/prepare/address_space.rs @@ -9,12 +9,12 @@ use axdevice_base::Resource; use super::super::{AxVM, AxVMResources, VM_ASPACE_BASE, VM_ASPACE_SIZE}; use crate::layout::{GuestOwnedRegion, VmRegionKind, build_address_layout}; -pub(super) fn map_guest_address_space( +pub(crate) fn map_guest_address_space( vm: &AxVM, resources: &mut AxVMResources, devices: &AxVmDevices, + owned_regions: &[GuestOwnedRegion], ) -> AxResult { - let owned_regions = guest_owned_regions(resources); let emulated_resources = devices .devices() .flat_map(|device| device.resources().iter().cloned()) @@ -25,7 +25,7 @@ pub(super) fn map_guest_address_space( VM_ASPACE_SIZE, resources.config.pass_through_devices(), resources.config.pass_through_addresses(), - &owned_regions, + owned_regions, &emulated_resources, )?; @@ -49,12 +49,10 @@ pub(super) fn map_guest_address_space( } resources.address_layout = Some(address_layout); - crate::arch::map_arch_address_space(&mut resources.address_space)?; - Ok(()) } -fn guest_owned_regions(resources: &AxVMResources) -> Vec { +pub(crate) fn guest_owned_regions(resources: &AxVMResources) -> Vec { let mut regions = resources .memory_regions .iter() @@ -81,7 +79,5 @@ fn guest_owned_regions(resources: &AxVMResources) -> Vec { }), ); - crate::arch::append_arch_owned_regions(&mut regions); - regions } diff --git a/virtualization/axvm/src/vm/prepare/devices.rs b/virtualization/axvm/src/vm/prepare/devices.rs index 3db1319ffe..5f24eee57a 100644 --- a/virtualization/axvm/src/vm/prepare/devices.rs +++ b/virtualization/axvm/src/vm/prepare/devices.rs @@ -6,19 +6,18 @@ use axdevice::{AxVmDeviceConfig, AxVmDevices, DeviceBuildContext, DeviceFactoryR use super::super::{AxVM, AxVMResources}; use crate::irq::InterruptFabric; -pub(super) struct PreparedDevices { - devices: AxVmDevices, +pub(crate) struct PreparedDevices { + pub(crate) devices: AxVmDevices, } impl PreparedDevices { - pub(super) fn build( - vm: &AxVM, + pub(crate) fn build_common( resources: &AxVMResources, factories: &DeviceFactoryRegistry, interrupt_fabric: &InterruptFabric, ) -> AxResult { let build_context = DeviceBuildContext::new(interrupt_fabric); - let mut devices = AxVmDevices::build_with_factories( + let devices = AxVmDevices::build_with_factories( AxVmDeviceConfig { emu_configs: resources.config.emu_devices().to_vec(), }, @@ -26,16 +25,18 @@ impl PreparedDevices { &build_context, )?; - crate::arch::register_arch_devices(vm, &resources.config, &mut devices)?; - vm.add_special_emulated_devices(&mut devices)?; Ok(Self { devices }) } - pub(super) const fn devices(&self) -> &AxVmDevices { + pub(crate) fn register_special_devices(&mut self, vm: &AxVM) -> AxResult { + vm.add_special_emulated_devices(&mut self.devices) + } + + pub(crate) const fn devices(&self) -> &AxVmDevices { &self.devices } - pub(super) fn into_inner(self) -> AxVmDevices { + pub(crate) fn into_inner(self) -> AxVmDevices { self.devices } } diff --git a/virtualization/axvm/src/vm/prepare/vcpus.rs b/virtualization/axvm/src/vm/prepare/vcpus.rs index f2e948cacc..2012650198 100644 --- a/virtualization/axvm/src/vm/prepare/vcpus.rs +++ b/virtualization/axvm/src/vm/prepare/vcpus.rs @@ -1,41 +1,41 @@ -//! vCPU construction and setup for VM preparation. +//! Architecture-neutral vCPU collection construction and setup. use alloc::{boxed::Box, sync::Arc, vec::Vec}; use ax_errno::AxResult; -use axvm_types::GuestPhysAddr; +use axvm_types::VmArchVcpuOps; use super::super::{AxVCpuRef, AxVMResources, VCpu}; -use crate::arch::{ArchOps, CurrentArch, VcpuCreateContext}; -pub(super) struct PreparedVcpus { +#[derive(Clone, Copy, Debug)] +pub(crate) struct VcpuPlacement { + pub(crate) id: usize, + pub(crate) phys_cpu_set: Option, + pub(crate) phys_cpu_id: usize, +} + +pub(crate) struct PreparedVcpus { vcpus: Vec, } impl PreparedVcpus { - pub(super) fn create( + pub(crate) fn create( vm_id: usize, - resources: &AxVMResources, - dtb_addr: Option, + placements: &[VcpuPlacement], + mut build_config: impl FnMut( + VcpuPlacement, + ) + -> AxResult<::CreateConfig>, ) -> AxResult { - let vcpu_id_pcpu_sets = resources.config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); - let create_state = CurrentArch::new_vcpu_create_state(&vcpu_id_pcpu_sets)?; - let firmware_boot = resources.config.uses_firmware_boot(); + debug!("id: {vm_id}, vCPU placements: {placements:#x?}"); - debug!("dtb_load_gpa: {dtb_addr:?}"); - debug!("id: {vm_id}, VCpuIdPCpuSets: {vcpu_id_pcpu_sets:#x?}"); - - let mut vcpus = Vec::with_capacity(vcpu_id_pcpu_sets.len()); - for (vcpu_id, phys_cpu_set, phys_cpu_id) in vcpu_id_pcpu_sets { - let arch_config = CurrentArch::build_vcpu_create_config( - &create_state, - VcpuCreateContext { - vcpu_id, - phys_cpu_id, - dtb_addr, - firmware_boot, - }, - )?; + let mut vcpus = Vec::with_capacity(placements.len()); + for placement in placements.iter().copied() { + trace!( + "Creating VM[{vm_id}] vCPU[{}] for physical CPU {}", + placement.id, placement.phys_cpu_id + ); + let arch_config = build_config(placement)?; // FIXME: VCpu is neither `Send` nor `Sync` by design, check whether // 1. we should make it `Send` and `Sync`, or @@ -43,8 +43,8 @@ impl PreparedVcpus { #[allow(clippy::arc_with_non_send_sync)] vcpus.push(Arc::new(VCpu::new( vm_id, - vcpu_id, - phys_cpu_set, + placement.id, + placement.phys_cpu_set, arch_config, )?)); } @@ -52,11 +52,17 @@ impl PreparedVcpus { Ok(Self { vcpus }) } - pub(super) fn setup(&self, resources: &AxVMResources) -> AxResult { + pub(crate) fn setup( + &self, + resources: &AxVMResources, + mut build_config: impl FnMut( + &crate::config::AxVMConfig, + &[crate::vm::VMMemoryRegion], + ) + -> AxResult<::SetupConfig>, + ) -> AxResult { for vcpu in &self.vcpus { - let setup_config = - CurrentArch::build_vcpu_setup_config(&resources.config, &resources.memory_regions)?; - + let setup_config = build_config(&resources.config, &resources.memory_regions)?; let entry = if vcpu.id() == 0 { resources.config.bsp_entry() } else { @@ -64,13 +70,26 @@ impl PreparedVcpus { }; debug!("Setting up vCPU[{}] entry at {:#x}", vcpu.id(), entry); - vcpu.setup(entry, resources.nested_paging, setup_config)?; } Ok(()) } - pub(super) fn into_boxed_slice(self) -> Box<[AxVCpuRef]> { + pub(crate) fn into_boxed_slice(self) -> Box<[AxVCpuRef]> { self.vcpus.into_boxed_slice() } } + +pub(crate) fn vcpu_placements(resources: &AxVMResources) -> Vec { + resources + .config + .phys_cpu_ls + .get_vcpu_affinities_pcpu_ids() + .into_iter() + .map(|(id, phys_cpu_set, phys_cpu_id)| VcpuPlacement { + id, + phys_cpu_set, + phys_cpu_id, + }) + .collect() +} diff --git a/virtualization/axvm/tests/arch_boundary_contract.rs b/virtualization/axvm/tests/arch_boundary_contract.rs index 40546a3833..a7c76a4eb5 100644 --- a/virtualization/axvm/tests/arch_boundary_contract.rs +++ b/virtualization/axvm/tests/arch_boundary_contract.rs @@ -17,7 +17,6 @@ fn vm_core_does_not_handle_arch_local_exits() { let vm_rs = include_str!("../src/vm/mod.rs"); for forbidden in [ - "CurrentArch", "ArchOps", "CurrentArch::handle_vcpu_exit", "VcpuRunAction", @@ -30,6 +29,92 @@ fn vm_core_does_not_handle_arch_local_exits() { } } +#[test] +fn common_vm_initialization_only_uses_high_level_arch_entrypoints() { + let vm = include_str!("../src/vm/mod.rs"); + let preparation = include_str!("../src/vm/prepare.rs"); + let common_sources = [ + vm, + preparation, + include_str!("../src/vm/prepare/address_space.rs"), + include_str!("../src/vm/prepare/devices.rs"), + include_str!("../src/vm/prepare/vcpus.rs"), + ]; + + assert!(vm.contains("CurrentArch::create_vm_resources(config)")); + assert!(preparation.contains("CurrentArch::init_vm")); + + for source in &common_sources { + for line in source.lines().filter(|line| line.contains("CurrentArch::")) { + assert!( + line.contains("CurrentArch::create_vm_resources") + || line.contains("CurrentArch::init_vm"), + "common VM initialization calls a fine-grained architecture hook: {line}" + ); + } + } + + for forbidden in [ + "configure_interrupt_fabric", + "register_arch_devices", + "append_arch_owned_regions", + "map_arch_address_space", + "new_vcpu_create_state", + "build_vcpu_create_config", + "build_vcpu_setup_config", + ] { + assert!( + common_sources + .iter() + .all(|source| !source.contains(forbidden)), + "common VM initialization must not call architecture step hook: {forbidden}" + ); + } +} + +#[test] +fn every_architecture_owns_vm_resource_creation_and_initialization() { + for source in [ + include_str!("../src/arch/aarch64/vm.rs"), + include_str!("../src/arch/loongarch64/vm.rs"), + include_str!("../src/arch/riscv64/vm.rs"), + include_str!("../src/arch/x86_64/vm.rs"), + ] { + assert!(source.contains("fn create_vm_resources")); + assert!(source.contains("fn init_vm")); + } +} + +#[test] +fn custom_vm_init_inputs_cross_the_arch_boundary_unchanged() { + let preparation = include_str!("../src/vm/prepare.rs"); + assert!(preparation.contains("VmInitRequest::Provided")); + assert!(preparation.contains("factories,")); + assert!(preparation.contains("interrupt_fabric,")); + + for source in [ + include_str!("../src/arch/aarch64/vm.rs"), + include_str!("../src/arch/loongarch64/vm.rs"), + include_str!("../src/arch/riscv64/vm.rs"), + include_str!("../src/arch/x86_64/vm.rs"), + ] { + assert!(source.contains("VmInitRequest::Provided")); + assert!(source.contains("init_vm_with(vm, factories, interrupt_fabric)")); + } +} + +#[test] +fn failed_vm_initialization_resets_transient_resources_before_retry() { + let preparation = include_str!("../src/vm/prepare.rs"); + let initialize = preparation + .split_once("let prepared = match initialize") + .expect("VM initialization must handle architecture errors") + .1; + + assert!(initialize.contains("resources.reset_transient_resources()")); + assert!(initialize.contains("return Err(err)")); +} + #[test] fn runtime_vcpu_loop_only_consumes_scheduler_actions() { let runtime_vcpus_rs = include_str!("../src/runtime/vcpus.rs"); @@ -226,6 +311,36 @@ fn vcpu_setup_context_keeps_named_capabilities() { ); } +#[test] +fn vm_init_capability_traits_are_not_reintroduced() { + let capabilities = include_str!("../src/architecture/capabilities.rs"); + let ops = include_str!("../src/architecture/ops.rs"); + + for forbidden in [ + "trait DevicePlatform", + "trait AddressSpacePlatform", + "VcpuCreateContext", + "fn build_vcpu_create_config", + "fn build_vcpu_setup_config", + ] { + assert!( + !capabilities.contains(forbidden) && !ops.contains(forbidden), + "VM initialization detail must stay behind CurrentArch::init_vm: {forbidden}" + ); + } +} + +#[test] +fn eager_vm_lifecycle_has_no_uninit_state() { + let status = include_str!("../src/lifecycle/status.rs"); + let machine = include_str!("../src/lifecycle/machine.rs"); + let vm = include_str!("../src/vm/mod.rs"); + + assert!(!status.contains("Uninit")); + assert!(!machine.contains("Machine::Uninit")); + assert!(vm.contains("machine: Mutex::new(Machine::Ready(resources))")); +} + fn find_target_arch_cfg_outside_arch( source_root: &std::path::Path, directory: &std::path::Path,