diff --git a/os/axvisor/src/config.rs b/os/axvisor/src/config.rs index 2dcc1d7bae..619da69ef5 100644 --- a/os/axvisor/src/config.rs +++ b/os/axvisor/src/config.rs @@ -22,11 +22,12 @@ use core::sync::atomic::{AtomicBool, Ordering}; use ax_errno::{AxResult, ax_err_type}; #[cfg(all(feature = "fs", target_arch = "x86_64"))] use axvm::InterruptTriggerMode; -#[cfg(target_arch = "x86_64")] -use axvm::config::VMBootProtocol; use axvm::{ AxVM, GuestPhysAddr, - boot::{BootImageProvider, ImageLoader, StaticVmImage, get_image_header}, + boot::{ + BootImageProvider, StaticVmImage, boot_firmware_load_gpa, get_image_header, + guest_boot_policy, init_guest_boot_resources, prepare_guest_boot, + }, config::{ AxVCpuConfig, AxVMConfig, AxVMConfigParams, GuestBootPolicy, PhysCpuList, RamdiskInfo, VMImageConfig, @@ -34,17 +35,6 @@ use axvm::{ }; use axvmconfig::{AxVMCrateConfig, VMType}; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use axvm::boot::handle_fdt_operations; -#[cfg(target_arch = "x86_64")] -use axvm::boot::is_x86_linux_image_config; -#[cfg(target_arch = "loongarch64")] -use axvm::boot::{handle_fdt_operations, init_guest_boot_resources}; - -/// Default BIOS load GPA for x86_64 built-in BIOS. -#[cfg(target_arch = "x86_64")] -const DEFAULT_X86_BIOS_LOAD_GPA: usize = 0x8000; - #[cfg(all( feature = "fs", any(target_arch = "x86_64", target_arch = "loongarch64") @@ -88,11 +78,7 @@ pub mod vmcfg { } pub fn init_guest_vms() { - // Initialize LoongArch firmware resources before guest configs are materialized. - #[cfg(target_arch = "loongarch64")] - { - init_guest_boot_resources(); - } + init_guest_boot_resources(); // First try to get configs from filesystem if fs feature is enabled let mut gvm_raw_configs = vmcfg::filesystem_vm_configs(); @@ -120,8 +106,7 @@ pub fn init_guest_vms() { pub fn init_guest_vm(raw_cfg: &str) -> AxResult { let image_provider = AxvisorBootImageProvider; - #[allow(unused_mut)] - let mut vm_create_config = AxVMCrateConfig::from_toml(raw_cfg) + let vm_create_config = AxVMCrateConfig::from_toml(raw_cfg) .map_err(|e| ax_err_type!(InvalidData, format!("Failed to resolve VM config: {e:?}")))?; #[cfg(all( @@ -137,25 +122,13 @@ pub fn init_guest_vm(raw_cfg: &str) -> AxResult { ); } - #[allow(unused_mut)] let mut vm_config = build_axvm_config(&vm_create_config); + let prepared_boot = prepare_guest_boot(&mut vm_config, vm_create_config, &image_provider)?; + let prepared_config = prepared_boot.config(); - // Handle FDT-related operations for architectures that boot guests with DTB. - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - let guest_dtb = handle_fdt_operations(&mut vm_config, &mut vm_create_config, &image_provider)?; - #[cfg(target_arch = "loongarch64")] - handle_fdt_operations(&mut vm_config, &mut vm_create_config)?; + sync_axvm_config_from_crate_config(&mut vm_config, prepared_config); - sync_axvm_config_from_crate_config(&mut vm_config, &vm_create_config); - - #[cfg(target_arch = "x86_64")] - let skip_guest_address_adjustment = x86_linux_direct_boot_config(&vm_create_config); - #[cfg(not(target_arch = "x86_64"))] - let skip_guest_address_adjustment = false; - vm_config.set_boot_policy(guest_boot_policy( - &vm_create_config, - skip_guest_address_adjustment, - )); + vm_config.set_boot_policy(guest_boot_policy(prepared_config, &image_provider)); // info!("after parse_vm_interrupt, crate VM[{}] with config: {:#?}", vm_config.id(), vm_config); info!("Creating VM[{}] {:?}", vm_config.id(), vm_config.name()); @@ -171,17 +144,7 @@ pub fn init_guest_vm(raw_cfg: &str) -> AxResult { // Load corresponding images for VM. info!("VM[{}] created success, loading images...", vm.id()); - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - let mut loader = ImageLoader::new( - main_mem, - vm_create_config, - vm.clone(), - &image_provider, - guest_dtb, - ); - #[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64")))] - let mut loader = ImageLoader::new(main_mem, vm_create_config, vm.clone(), &image_provider); - loader.load()?; + prepared_boot.load_images(main_mem, vm.clone(), &image_provider)?; vm.prepare() .map_err(|e| ax_err_type!(InvalidData, format!("VM[{}] setup failed: {e:?}", vm.id())))?; @@ -225,7 +188,7 @@ pub(crate) fn build_axvm_config(cfg: &AxVMCrateConfig) -> AxVMConfig { image_config: VMImageConfig { kernel_load_gpa: GuestPhysAddr::from(cfg.kernel.kernel_load_addr), loaded_from_filesystem: cfg.kernel.image_location.as_deref() == Some("fs"), - bios_load_gpa: configured_bios_load_gpa(cfg), + bios_load_gpa: boot_firmware_load_gpa(cfg), dtb_load_gpa: cfg.kernel.dtb_load_addr.map(GuestPhysAddr::from), ramdisk: cfg.kernel.ramdisk_load_addr.map(|addr| RamdiskInfo { load_gpa: GuestPhysAddr::from(addr), @@ -249,38 +212,6 @@ fn sync_axvm_config_from_crate_config(vm_config: &mut AxVMConfig, cfg: &AxVMCrat vm_config.set_memory_regions(cfg.kernel.memory_regions.clone()); } -fn guest_boot_policy( - cfg: &AxVMCrateConfig, - skip_guest_address_adjustment: bool, -) -> GuestBootPolicy { - if skip_guest_address_adjustment { - GuestBootPolicy::KeepConfigured - } else { - GuestBootPolicy::AdjustKernelForBootProtocol { - protocol: cfg.kernel.effective_boot_protocol(), - } - } -} - -fn configured_bios_load_gpa(cfg: &AxVMCrateConfig) -> Option { - if !cfg.kernel.enable_bios { - return None; - } - - if let Some(addr) = cfg.kernel.bios_load_addr { - return Some(GuestPhysAddr::from(addr)); - } - - #[cfg(target_arch = "x86_64")] - if cfg.kernel.boot_firmware_path().is_none() - && cfg.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot - { - return Some(GuestPhysAddr::from(DEFAULT_X86_BIOS_LOAD_GPA)); - } - - None -} - #[cfg(all( feature = "fs", any(target_arch = "x86_64", target_arch = "loongarch64") @@ -419,11 +350,6 @@ fn x86_intx_forwarding_trigger(binding: &ax_driver::BindingIrq) -> InterruptTrig } } -#[cfg(target_arch = "x86_64")] -fn x86_linux_direct_boot_config(config: &AxVMCrateConfig) -> bool { - is_x86_linux_image_config(config, &AxvisorBootImageProvider) -} - struct AxvisorBootImageProvider; impl BootImageProvider for AxvisorBootImageProvider { diff --git a/os/axvisor/src/shell/command/vm.rs b/os/axvisor/src/shell/command/vm.rs index 041130b187..75faf258c6 100644 --- a/os/axvisor/src/shell/command/vm.rs +++ b/os/axvisor/src/shell/command/vm.rs @@ -34,7 +34,6 @@ fn can_start_vm(status: VmStatus) -> Result<(), &'static str> { VmStatus::Running => Err("VM is already running"), VmStatus::Paused => Err("VM is suspended, use 'vm resume' instead"), VmStatus::Stopping => Err("VM is stopping, wait for it to fully stop"), - VmStatus::Uninit => Err("VM is still loading"), VmStatus::Pausing => Err("VM is pausing"), VmStatus::Destroying | VmStatus::Destroyed => Err("VM is being destroyed"), VmStatus::Failed => Err("VM is failed"), @@ -54,7 +53,7 @@ fn can_stop_vm(status: VmStatus, force: bool) -> Result<(), &'static str> { } } VmStatus::Stopped => Err("VM is already stopped"), - VmStatus::Uninit | VmStatus::Ready => Ok(()), // Allow stopping VMs in these states + VmStatus::Ready => Ok(()), // Allow stopping VMs before their first start. VmStatus::Pausing => Err("VM is pausing"), VmStatus::Destroying | VmStatus::Destroyed => Err("VM is being destroyed"), VmStatus::Failed => Err("VM is failed"), @@ -68,7 +67,6 @@ fn can_suspend_vm(status: VmStatus) -> Result<(), &'static str> { VmStatus::Paused => Err("VM is already suspended"), VmStatus::Stopped => Err("VM is stopped, cannot suspend"), VmStatus::Stopping => Err("VM is stopping, cannot suspend"), - VmStatus::Uninit => Err("VM is loading, cannot suspend"), VmStatus::Ready => Err("VM is not running, cannot suspend"), VmStatus::Pausing => Err("VM is already pausing"), VmStatus::Destroying | VmStatus::Destroyed => Err("VM is being destroyed"), @@ -83,7 +81,6 @@ fn can_resume_vm(status: VmStatus) -> Result<(), &'static str> { VmStatus::Running => Err("VM is already running"), VmStatus::Stopped => Err("VM is stopped, use 'vm start' instead"), VmStatus::Stopping => Err("VM is stopping, cannot resume"), - VmStatus::Uninit => Err("VM is loading, cannot resume"), VmStatus::Ready => Err("VM is not started yet, use 'vm start' instead"), VmStatus::Pausing => Err("VM is pausing, wait before resuming"), VmStatus::Destroying | VmStatus::Destroyed => Err("VM is being destroyed"), @@ -305,7 +302,7 @@ fn stop_vm_by_id(vm_id: usize, force: bool) { println!("Gracefully stopping VM[{}]...", vm_id); } } - VmStatus::Uninit | VmStatus::Ready => { + VmStatus::Ready => { println!( "⚠ VM[{}] is in {:?} state, stopping anyway...", vm_id, status diff --git a/virtualization/axvm/src/arch/aarch64/capabilities.rs b/virtualization/axvm/src/arch/aarch64/capabilities.rs new file mode 100644 index 0000000000..b232555ad7 --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/capabilities.rs @@ -0,0 +1,86 @@ +//! AArch64 implementations of AxVM platform capability hooks. + +use alloc::format; + +use ax_errno::{AxResult, ax_err_type}; + +use super::Aarch64Arch; +use crate::architecture::{BootImagePlatform, GuestBootPlatform, HostTimePlatform}; + +impl HostTimePlatform for Aarch64Arch {} + +impl BootImagePlatform for Aarch64Arch { + fn load_guest_dtb( + loader: &crate::boot::images::ImageLoaderCore<'_>, + dtb: &crate::boot::fdt::GuestDtbImage, + ) -> AxResult { + let bytes = dtb.as_bytes(); + let source = core::ptr::NonNull::new(bytes.as_ptr() as *mut u8) + .ok_or_else(|| ax_err_type!(InvalidData, "Guest DTB pointer is null"))?; + super::fdt::core::update_fdt(source, bytes.len(), loader.vm.clone(), &loader.config) + } +} + +impl GuestBootPlatform for Aarch64Arch { + fn prepare_guest_boot( + vm_config: &mut crate::config::AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn crate::boot::BootImageProvider, + ) -> AxResult> { + super::fdt::core::prepare_dtb_guest(vm_config, vm_create_config, provider) + } +} + +pub fn host_fdt_bootarg() -> usize { + ax_std::os::arceos::modules::ax_hal::dtb::get_bootarg() +} + +pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + ax_std::os::arceos::modules::ax_hal::mem::phys_to_virt(paddr) +} + +pub(super) fn decode_gic_spi(specifier: &[u32]) -> Option { + (specifier.first().copied() == Some(0)) + .then(|| specifier.get(1).copied()) + .flatten() +} + +pub(super) fn patch_runtime_fdt( + fdt_bytes: &[u8], + vm: &crate::AxVMRef, + crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + let initrd = vm.with_config(|config| { + super::fdt::initrd_start_size_from_image_config(config.image_config.ramdisk.as_ref()) + }); + super::fdt::core::patch_guest_fdt_for_runtime( + fdt_bytes, + &vm.memory_regions(), + crate_config, + initrd, + true, + ) +} + +pub(super) fn patch_provided_fdt( + provided_dtb: &[u8], + host_dtb: Option<&[u8]>, + crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + let provided_fdt = fdt_edit::Fdt::from_bytes(provided_dtb).map_err(|err| { + ax_err_type!( + InvalidData, + format!("Failed to parse provided DTB image: {err:#?}") + ) + })?; + let host_fdt = host_dtb + .map(fdt_edit::Fdt::from_bytes) + .transpose() + .map_err(|err| { + ax_err_type!( + InvalidData, + format!("Failed to parse host DTB image: {err:#?}") + ) + })?; + super::fdt::update_cpu_node(&provided_fdt, host_fdt.as_ref(), crate_config) +} diff --git a/virtualization/axvm/src/arch/aarch64/fdt.rs b/virtualization/axvm/src/arch/aarch64/fdt.rs new file mode 100644 index 0000000000..a3c0db1fb5 --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/fdt.rs @@ -0,0 +1,97 @@ +//! AArch64 compatibility facade and target-specific guest FDT policy. + +use alloc::vec::Vec; + +use ax_errno::{AxResult, ax_err_type}; +use fdt_edit::Fdt; + +use crate::{ + boot::{BootImageProvider, fdt::GuestDtbImage}, + config::AxVMConfig, +}; + +#[path = "../../boot/fdt/core/mod.rs"] +pub(crate) mod core; + +pub use core::{ + parse_passthrough_devices_address, parse_reserved_memory_regions, parse_vm_interrupt, + reserve_excluded_device_ranges, set_phys_cpu_sets, setup_guest_fdt_from_vmm, try_get_host_fdt, + update_fdt, update_provided_fdt, +}; + +pub(crate) fn guest_fdt_policy() -> core::GuestFdtPolicy { + core::GuestFdtPolicy { + patch_runtime: super::capabilities::patch_runtime_fdt, + patch_provided: super::capabilities::patch_provided_fdt, + decode_interrupt: super::capabilities::decode_gic_spi, + } +} + +pub(crate) fn host_fdt_bootarg() -> usize { + super::capabilities::host_fdt_bootarg() +} + +pub(crate) fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + super::capabilities::host_phys_to_virt(paddr) +} + +pub(super) fn initrd_start_size_from_image_config( + ramdisk: Option<&crate::config::RamdiskInfo>, +) -> Option<(u64, u64)> { + let ramdisk = ramdisk?; + Some((ramdisk.load_gpa.as_usize() as u64, ramdisk.size? as u64)) +} + +pub(super) fn update_cpu_node( + fdt: &Fdt, + host_fdt: Option<&Fdt>, + crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + let Some(host_fdt) = host_fdt else { + return Ok(fdt.encode().as_ref().to_vec()); + }; + + let phys_cpu_ids = crate_config + .base + .phys_cpu_ids + .as_deref() + .ok_or_else(|| ax_err_type!(InvalidInput, "phys_cpu_ids is missing"))?; + let mut tree = core::tree::FdtTree::from_fdt(fdt.clone()); + tree.inner_mut().remove_by_path("/cpus"); + + if let Some(host_cpus_id) = host_fdt.get_by_path_id("/cpus") { + let cpus_id = + tree.copy_subtree_from(host_fdt, host_cpus_id, tree.inner().root_id(), true)?; + let cpu_paths = tree + .node_paths() + .into_iter() + .filter_map(|(id, path)| { + (path.starts_with("/cpus/cpu@") + && !core::create::need_cpu_node(phys_cpu_ids, tree.inner(), id, &path)) + .then_some(path) + }) + .collect::>(); + for path in cpu_paths { + tree.inner_mut().remove_by_path(&path); + } + if let Some(cpus) = tree.inner_mut().node_mut(cpus_id) { + for property in [ + "riscv,cbop-block-size", + "riscv,cboz-block-size", + "riscv,cbom-block-size", + ] { + cpus.remove_property(property); + } + } + } + + Ok(tree.finish()) +} + +pub fn handle_fdt_operations( + vm_config: &mut AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult> { + core::prepare_dtb_guest(vm_config, vm_create_config, provider) +} diff --git a/virtualization/axvm/src/host/gic.rs b/virtualization/axvm/src/arch/aarch64/gic.rs similarity index 97% rename from virtualization/axvm/src/host/gic.rs rename to virtualization/axvm/src/arch/aarch64/gic.rs index 1c0e53d869..5f97b8c5c8 100644 --- a/virtualization/axvm/src/host/gic.rs +++ b/virtualization/axvm/src/arch/aarch64/gic.rs @@ -6,7 +6,7 @@ use arm_gic_driver::v3::{ }; use ax_memory_addr::{PhysAddr, VirtAddr}; -use super::{HostMemory, arceos, default_host}; +use crate::host::{HostMemory, default_host}; fn with_gic(f: impl FnOnce(&mut rdif_intc::Intc) -> T) -> T { let mut gic = rdrive::get_one::() @@ -140,7 +140,7 @@ pub(crate) fn handle_current_irq() -> Option { // AArch64 ArceOS platform IRQ handlers acknowledge the current IRQ // internally. The raw vector argument is ignored by current GIC-backed // platforms, so keep the ack/EOI ownership inside the platform handler. - arceos::handle_host_irq(0) + ax_std::os::arceos::modules::ax_hal::irq::handle_irq(0).then_some(0) } pub(crate) fn fetch_irq() -> usize { diff --git a/virtualization/axvm/src/arch/aarch64/images.rs b/virtualization/axvm/src/arch/aarch64/images.rs new file mode 100644 index 0000000000..f2f256817f --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/images.rs @@ -0,0 +1,33 @@ +//! Public AArch64 image-loader facade preserving the DTB constructor contract. + +use ax_errno::AxResult; +use axvmconfig::AxVMCrateConfig; + +use crate::{ + AxVMRef, VMMemoryRegion, + boot::{BootImageProvider, fdt::GuestDtbImage, images::ImageLoaderCore}, +}; + +pub struct ImageLoader<'a>(ImageLoaderCore<'a>); + +impl<'a> ImageLoader<'a> { + pub fn new( + main_memory: VMMemoryRegion, + config: AxVMCrateConfig, + vm: AxVMRef, + provider: &'a dyn BootImageProvider, + guest_dtb: Option, + ) -> Self { + Self(ImageLoaderCore::new( + main_memory, + config, + vm, + provider, + guest_dtb, + )) + } + + pub fn load(&mut self) -> AxResult { + self.0.load() + } +} diff --git a/virtualization/axvm/src/arch/aarch64/ipi.rs b/virtualization/axvm/src/arch/aarch64/ipi.rs new file mode 100644 index 0000000000..09f99fcc04 --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/ipi.rs @@ -0,0 +1,90 @@ +//! AArch64 guest IPI target resolution and injection. + +use ax_errno::AxResult; + +use super::Aarch64DeferredRunWork; +use crate::architecture::BoundVcpuExit; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct SendIpiExit { + pub(crate) target_cpu: u64, + pub(crate) target_cpu_aux: u64, + pub(crate) send_to_all: bool, + pub(crate) send_to_self: bool, + pub(crate) vector: u64, +} + +pub(crate) fn handle( + vm: &crate::AxVMRef, + vcpu_id: usize, + exit: SendIpiExit, +) -> AxResult> { + let vm_id = vm.id(); + debug!( + "VM[{vm_id}] run VCpu[{vcpu_id}] SendIPI, target_cpu={:#x}, target_cpu_aux={:#x}, \ + vector={}", + exit.target_cpu, exit.target_cpu_aux, exit.vector + ); + let targets = ipi_targets(vm, vcpu_id, exit); + if targets.is_empty() { + warn!( + "VM[{vm_id}] SendIPI has no target: target_cpu={:#x}, target_cpu_aux={:#x}", + exit.target_cpu, exit.target_cpu_aux + ); + return Ok(BoundVcpuExit::Complete( + crate::architecture::VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }, + )); + } + + if targets.get(vcpu_id) { + crate::inject_current_vcpu_interrupt(exit.vector as _) + .expect("failed to inject self IPI into current vCPU"); + } + let mut remote_targets = targets; + remote_targets.set(vcpu_id, false); + if !remote_targets.is_empty() + && let Err(err) = vm.inject_interrupt_to_vcpu(remote_targets, exit.vector as _) + { + warn!( + "Failed to inject interrupt {} to VM[{vm_id}] targets {remote_targets:?}: {err:?}", + exit.vector + ); + } + Ok(BoundVcpuExit::Complete( + crate::architecture::VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }, + )) +} + +fn ipi_targets( + vm: &crate::AxVMRef, + current_vcpu_id: usize, + exit: SendIpiExit, +) -> crate::CpuMask<64> { + let mut targets = crate::CpuMask::new(); + if exit.send_to_all { + for vcpu in vm.vcpu_list() { + if vcpu.id() != current_vcpu_id { + targets.set(vcpu.id(), true); + } + } + } else if exit.send_to_self { + targets.set(current_vcpu_id, true); + } else { + for (vcpu_id, _, phys_id) in vm.get_vcpu_affinities_pcpu_ids() { + let affinity = phys_id as u64; + let aff0 = affinity & 0xff; + let aff123 = affinity & !0xff; + if aff123 == exit.target_cpu && aff0 < 16 && (exit.target_cpu_aux & (1u64 << aff0)) != 0 + { + targets.set(vcpu_id, true); + } + } + } + targets +} diff --git a/virtualization/axvm/src/arch/aarch64/mod.rs b/virtualization/axvm/src/arch/aarch64/mod.rs index a1600e21e9..702663fb88 100644 --- a/virtualization/axvm/src/arch/aarch64/mod.rs +++ b/virtualization/axvm/src/arch/aarch64/mod.rs @@ -20,14 +20,26 @@ use axvm_types::{ VmArchVcpuOps, }; -use super::{ - ArchOps, BoundVcpuExit, CpuUpExit, HypercallExit, MmioReadExit, MmioWriteExit, SendIpiExit, - SysRegReadExit, SysRegWriteExit, VcpuCreateContext, VcpuRunAction, VcpuSetupContext, - target_phys_cpu_ids, -}; -use crate::host::{HostCpu, HostMemory, HostTime, default_host, gic}; - +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; +use crate::host::{HostCpu, HostMemory, HostTime, default_host}; + +mod capabilities; +#[path = "../../architecture/cpu_up.rs"] +mod cpu_up; +pub(crate) mod fdt; +mod gic; +mod images; +mod ipi; mod npt; +#[path = "../../architecture/sysreg.rs"] +mod sysreg; +mod vm; + +pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; +use cpu_up::{CpuUpExit, CpuUpOps}; +pub use images::ImageLoader; +use ipi::SendIpiExit; +use sysreg::{SysRegReadExit, SysRegWriteExit}; pub(crate) struct Aarch64Arch; @@ -36,10 +48,11 @@ pub(crate) enum Aarch64DeferredRunWork { ExternalInterrupt { vector: usize }, } +impl CpuUpOps for Aarch64Arch {} + impl ArchOps for Aarch64Arch { type VCpu = AxvmArmVcpu; type PerCpu = AxvmArmPerCpu; - type VcpuCreateState = (); type DeferredRunWork = Aarch64DeferredRunWork; type NestedPageTable = npt::NestedPageTable; @@ -47,61 +60,6 @@ impl ArchOps for Aarch64Arch { arm_vcpu::has_hardware_support() } - fn max_guest_page_table_levels() -> usize { - arm_vcpu::max_guest_page_table_levels() - } - - fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { - let mut selected = usize::MAX; - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - let levels = crate::percpu::cpu_max_guest_page_table_levels(cpu_id) - .unwrap_or_else(arm_vcpu::max_guest_page_table_levels); - if levels == 0 { - return ax_err!( - Unsupported, - "AArch64 nested paging is not enabled on target CPU" - ); - } - selected = selected.min(levels); - } - if selected == usize::MAX { - selected = arm_vcpu::max_guest_page_table_levels(); - } - match selected { - 3 | 4 => Ok(selected), - _ => ax_err!(Unsupported, "unsupported AArch64 stage-2 page-table levels"), - } - } - - fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - let mut pa_bits = usize::MAX; - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - let bits = - crate::percpu::cpu_guest_phys_addr_bits(cpu_id).unwrap_or_else(arm_vcpu::pa_bits); - pa_bits = pa_bits.min(bits); - } - if pa_bits == usize::MAX { - pa_bits = arm_vcpu::pa_bits(); - } - - let gpa_bits = match levels { - 3 => 39, - 4 => 48, - _ => return ax_err!(InvalidInput, "unsupported AArch64 stage-2 levels"), - }; - Ok(NestedPagingConfig::new( - root_paddr, levels, gpa_bits, pa_bits, - )) - } - - fn new_nested_page_table(levels: usize) -> AxResult { - npt::NestedPageTable::new(levels) - } - fn clean_dcache_range(addr: VirtAddr, size: usize) { aarch64_cpu_ext::cache::dcache_range( aarch64_cpu_ext::cache::CacheOp::Clean, @@ -110,62 +68,6 @@ impl ArchOps for Aarch64Arch { ); } - fn new_vcpu_create_state( - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - Ok(()) - } - - fn build_vcpu_create_config( - _state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig> { - Ok(ArmVcpuCreateConfig { - mpidr_el1: ctx.phys_cpu_id as _, - dtb_addr: ctx.dtb_addr.unwrap_or_default().as_usize(), - }) - } - - fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, - ) -> AxResult<::SetupConfig> { - let passthrough = ctx.interrupt_mode == axvm_types::VMInterruptMode::Passthrough; - Ok(ArmVcpuSetupConfig { - passthrough_interrupt: passthrough, - passthrough_timer: passthrough, - }) - } - - fn ipi_targets( - vm: &crate::AxVMRef, - current_vcpu_id: usize, - target_cpu: u64, - target_cpu_aux: u64, - send_to_all: bool, - send_to_self: bool, - ) -> crate::CpuMask<64> { - let mut targets = crate::CpuMask::new(); - if send_to_all { - for vcpu in vm.vcpu_list() { - if vcpu.id() != current_vcpu_id { - targets.set(vcpu.id(), true); - } - } - } else if send_to_self { - targets.set(current_vcpu_id, true); - } else { - for (vcpu_id, _, phys_id) in vm.get_vcpu_affinities_pcpu_ids() { - let affinity = phys_id as u64; - let aff0 = affinity & 0xff; - let aff123 = affinity & !0xff; - if aff123 == target_cpu && aff0 < 16 && (target_cpu_aux & (1u64 << aff0)) != 0 { - targets.set(vcpu_id, true); - } - } - } - targets - } - fn handle_vcpu_exit_bound( vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef, @@ -200,7 +102,7 @@ impl ArchOps for Aarch64Arch { data, }, ), - ArmVmExit::SysRegRead { addr, reg } => super::handle_sys_reg_read( + ArmVmExit::SysRegRead { addr, reg } => sysreg::handle_read( vm, vcpu, SysRegReadExit { @@ -208,7 +110,7 @@ impl ArchOps for Aarch64Arch { reg, }, ), - ArmVmExit::SysRegWrite { addr, value } => super::handle_sys_reg_write( + ArmVmExit::SysRegWrite { addr, value } => sysreg::handle_write( vm, SysRegWriteExit { addr: arm_sys_reg_addr_to_ax(addr), @@ -229,13 +131,16 @@ impl ArchOps for Aarch64Arch { vm.id(), vcpu.id() ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Wait)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: true, + stop_reason: None, + })) } ArmVmExit::CpuUp { target_cpu, entry_point, arg, - } => super::handle_cpu_up::( + } => cpu_up::handle::( vm, vcpu, CpuUpExit { @@ -246,9 +151,10 @@ impl ArchOps for Aarch64Arch { ), ArmVmExit::SystemDown => { warn!("VM[{}] run VCpu[{}] SystemDown", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Stop( - crate::StopReason::SystemDown, - ))) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: Some(crate::StopReason::SystemDown), + })) } ArmVmExit::SendIPI { target_cpu, @@ -256,7 +162,7 @@ impl ArchOps for Aarch64Arch { send_to_all, send_to_self, vector, - } => super::handle_send_ipi::( + } => ipi::handle( vm, vcpu.id(), SendIpiExit { @@ -267,7 +173,10 @@ impl ArchOps for Aarch64Arch { vector, }, ), - ArmVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)), + ArmVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })), _ => ax_err!(Unsupported, "unsupported AArch64 VM exit"), } } @@ -282,7 +191,10 @@ impl ArchOps for Aarch64Arch { Self::after_external_interrupt(vm, vcpu, vector); } } - Ok(VcpuRunAction::Yield) + Ok(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }) } } @@ -498,7 +410,7 @@ impl ArmVgicHostIf for ArmVgicHostIfImpl { } fn register_timer(deadline: Duration, callback: Box) { - let _ = default_host().register_timer(deadline.as_nanos() as u64, callback); + let _ = crate::timer::register_timer(deadline.as_nanos() as u64, callback); } fn read_vgicd_iidr() -> u32 { diff --git a/virtualization/axvm/src/arch/aarch64/npt.rs b/virtualization/axvm/src/arch/aarch64/npt.rs index f480db919b..d0a05a445b 100644 --- a/virtualization/axvm/src/arch/aarch64/npt.rs +++ b/virtualization/axvm/src/arch/aarch64/npt.rs @@ -231,7 +231,7 @@ impl ptg::TableMeta for A64HVPagingMetaDataL4 { } pub(crate) type NestedPageTable = - crate::arch::npt::LeveledPageTable; + crate::npt::LeveledPageTable; fn config_to_flags(config: ptg::PteConfig) -> MappingFlags { let mut flags = MappingFlags::empty(); diff --git a/virtualization/axvm/src/arch/aarch64/vm.rs b/virtualization/axvm/src/arch/aarch64/vm.rs new file mode 100644 index 0000000000..43b3972f18 --- /dev/null +++ b/virtualization/axvm/src/arch/aarch64/vm.rs @@ -0,0 +1,177 @@ +//! AArch64 VM resource creation and initialization. + +use alloc::sync::Arc; + +use arm_vcpu::{ArmVcpuCreateConfig, ArmVcpuSetupConfig}; +use ax_errno::{AxResult, ax_err, ax_err_type}; +use axdevice_base::DeviceRegistry as _; +use axvm_types::{NestedPagingConfig, VMInterruptMode, VmArchVcpuOps}; + +use super::{Aarch64Arch, npt}; +use crate::{ + config::AxVMConfig, + vm::{ + AxVM, AxVMResources, + prepare::{ + PreparedVm, VmInitRequest, + address_space::{guest_owned_regions, map_guest_address_space}, + complete_vm_init, default_device_factories, + devices::PreparedDevices, + validate_guest_dtb, + vcpus::{PreparedVcpus, vcpu_placements}, + }, + }, +}; + +impl Aarch64Arch { + pub(crate) fn create_vm_resources(config: AxVMConfig) -> AxResult { + let placements = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); + let levels = guest_page_table_levels(&placements)?; + let page_table = npt::NestedPageTable::new(levels)?; + AxVMResources::from_page_table(config, page_table, |root_paddr| { + nested_paging_config(root_paddr, levels, &placements) + }) + } + + pub(crate) fn init_vm(vm: &AxVM, request: VmInitRequest<'_>) -> AxResult { + match request { + VmInitRequest::Default => { + let factories = default_device_factories()?; + let interrupt_fabric = crate::InterruptFabric::new(vm.interrupt_mode()); + init_vm_with(vm, &factories, interrupt_fabric) + } + VmInitRequest::Provided { + factories, + interrupt_fabric, + } => init_vm_with(vm, factories, interrupt_fabric), + } + } +} + +fn init_vm_with( + vm: &AxVM, + factories: &axdevice::DeviceFactoryRegistry, + interrupt_fabric: crate::InterruptFabric, +) -> AxResult { + complete_vm_init(vm, interrupt_fabric, |resources, interrupt_fabric| { + let placements = vcpu_placements(resources); + let dtb_addr = resources + .config() + .image_config() + .dtb_load_gpa + .unwrap_or_default(); + let vcpus = PreparedVcpus::create(vm.id(), &placements, |placement| { + Ok(ArmVcpuCreateConfig { + mpidr_el1: placement.phys_cpu_id as _, + dtb_addr: dtb_addr.as_usize(), + }) + })?; + let mut devices = PreparedDevices::build_common(resources, factories, interrupt_fabric)?; + register_arch_devices(vm, resources.config(), &mut devices.devices)?; + devices.register_special_devices(vm)?; + validate_guest_dtb(resources)?; + + let owned_regions = guest_owned_regions(resources); + map_guest_address_space(vm, resources, devices.devices(), &owned_regions)?; + vcpus.setup(resources, build_vcpu_setup_config)?; + + Ok(PreparedVm::new(vcpus, devices)) + }) +} + +fn build_vcpu_setup_config( + config: &AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], +) -> AxResult<::SetupConfig> { + let passthrough = config.interrupt_mode() == VMInterruptMode::Passthrough; + Ok(ArmVcpuSetupConfig { + passthrough_interrupt: passthrough, + passthrough_timer: passthrough, + }) +} + +fn register_arch_devices( + vm: &AxVM, + config: &AxVMConfig, + devices: &mut axdevice::AxVmDevices, +) -> AxResult { + if config.interrupt_mode() == VMInterruptMode::Passthrough { + assign_passthrough_spis(vm, config, devices); + } else { + register_virtual_timers(devices)?; + } + Ok(()) +} + +fn assign_passthrough_spis(vm: &AxVM, config: &AxVMConfig, devices: &axdevice::AxVmDevices) { + let cpu_id = vm.id() - 1; // FIXME: get the real CPU id. + let Some(gicd) = devices + .devices() + .find_map(|device| device.as_any().downcast_ref::()) + else { + warn!("Failed to assign SPIs: No VGicD found in device list"); + return; + }; + + for spi in config.pass_through_spis() { + gicd.assign_irq(*spi + 32, cpu_id, (0, 0, 0, cpu_id as _)); + } +} + +fn register_virtual_timers(devices: &mut axdevice::AxVmDevices) -> AxResult { + for device in axdevice::create_vtimer_devices() { + devices + .register(Arc::from(device) as Arc) + .map_err(|err| { + ax_err_type!(InvalidInput, alloc::format!("register vtimer: {err:?}")) + })?; + } + Ok(()) +} + +fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { + let mut selected = usize::MAX; + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + let levels = crate::percpu::cpu_max_guest_page_table_levels(cpu_id) + .unwrap_or_else(arm_vcpu::max_guest_page_table_levels); + if levels == 0 { + return ax_err!( + Unsupported, + "AArch64 nested paging is not enabled on target CPU" + ); + } + selected = selected.min(levels); + } + if selected == usize::MAX { + selected = arm_vcpu::max_guest_page_table_levels(); + } + match selected { + 3 | 4 => Ok(selected), + _ => ax_err!(Unsupported, "unsupported AArch64 stage-2 page-table levels"), + } +} + +fn nested_paging_config( + root_paddr: ax_memory_addr::PhysAddr, + levels: usize, + vcpu_mappings: &[(usize, Option, usize)], +) -> AxResult { + let mut pa_bits = usize::MAX; + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + let bits = + crate::percpu::cpu_guest_phys_addr_bits(cpu_id).unwrap_or_else(arm_vcpu::pa_bits); + pa_bits = pa_bits.min(bits); + } + if pa_bits == usize::MAX { + pa_bits = arm_vcpu::pa_bits(); + } + + let gpa_bits = match levels { + 3 => 39, + 4 => 48, + _ => return ax_err!(InvalidInput, "unsupported AArch64 stage-2 levels"), + }; + Ok(NestedPagingConfig::new( + root_paddr, levels, gpa_bits, pa_bits, + )) +} diff --git a/virtualization/axvm/src/boot/fdt/loongarch64/guest_firmware_dtb.rs b/virtualization/axvm/src/arch/loongarch64/boot/fdt/guest_firmware_dtb.rs similarity index 99% rename from virtualization/axvm/src/boot/fdt/loongarch64/guest_firmware_dtb.rs rename to virtualization/axvm/src/arch/loongarch64/boot/fdt/guest_firmware_dtb.rs index 5d7c3ca36e..dc8ecab222 100644 --- a/virtualization/axvm/src/boot/fdt/loongarch64/guest_firmware_dtb.rs +++ b/virtualization/axvm/src/arch/loongarch64/boot/fdt/guest_firmware_dtb.rs @@ -4,7 +4,7 @@ use ax_errno::{AxResult, ax_err_type}; use fdt_edit::{Fdt, Node, NodeId}; use super::property::{prop_null, prop_string, prop_string_list, prop_u32, prop_u32_array}; -use crate::boot::guest_platform::loongarch64::GuestPlatform; +use crate::arch::guest_platform::GuestPlatform; const PHANDLE_CPU0: u32 = 0x8000; const PHANDLE_CPUIC: u32 = 0x8001; diff --git a/virtualization/axvm/src/boot/fdt/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/boot/fdt/mod.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/loongarch64/mod.rs rename to virtualization/axvm/src/arch/loongarch64/boot/fdt/mod.rs diff --git a/virtualization/axvm/src/boot/fdt/loongarch64/property.rs b/virtualization/axvm/src/arch/loongarch64/boot/fdt/property.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/loongarch64/property.rs rename to virtualization/axvm/src/arch/loongarch64/boot/fdt/property.rs diff --git a/virtualization/axvm/src/boot/guest_platform/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/boot/mod.rs similarity index 55% rename from virtualization/axvm/src/boot/guest_platform/loongarch64/mod.rs rename to virtualization/axvm/src/arch/loongarch64/boot/mod.rs index 4b17fba3e8..250234803e 100644 --- a/virtualization/axvm/src/boot/guest_platform/loongarch64/mod.rs +++ b/virtualization/axvm/src/arch/loongarch64/boot/mod.rs @@ -1,22 +1,53 @@ +mod fdt; mod probe; mod resources; -use alloc::{boxed::Box, vec::Vec}; +use alloc::{boxed::Box, format, vec::Vec}; use ax_errno::{AxResult, ax_err_type}; use axdevice::{ FwCfgInterruptConfig, FwCfgPciConfig, FwCfgPlatformConfig, FwCfgRamRegion, FwCfgSerialConfig, }; -use axvmconfig::{AxVMCrateConfig, EmulatedDeviceType}; +use axvmconfig::{AxVMCrateConfig, EmulatedDeviceType, VMBootProtocol}; pub use resources::{ LoongArchGuestIrqRoute, get_guest_irq_routes, prepare_uefi_fdt_config, prepare_uefi_runtime_config, }; -use crate::{AxVMRef, GuestPhysAddr, boot::images::load_vm_image_from_memory}; +use crate::{ + AxVMRef, GuestPhysAddr, + architecture::BootImagePlatform, + boot::{ + BootImageProvider, StaticVmImage, + images::{ImageLoaderCore, load_vm_image_from_memory}, + }, +}; pub const UEFI_FIRMWARE_FDT_BASE: usize = 0x0010_0000; +pub struct ImageLoader<'a>(ImageLoaderCore<'a>); + +impl<'a> ImageLoader<'a> { + pub fn new( + main_memory: crate::VMMemoryRegion, + config: AxVMCrateConfig, + vm: AxVMRef, + provider: &'a dyn BootImageProvider, + ) -> Self { + Self(ImageLoaderCore::new( + main_memory, + config, + vm, + provider, + None, + )) + } + + pub fn load(&mut self) -> AxResult { + self.0.load() + } +} + pub fn init() { resources::init(); } @@ -145,7 +176,7 @@ impl GuestPlatform { pub fn load_firmware_fdt(vm: &AxVMRef, config: &AxVMCrateConfig) -> AxResult { let platform = GuestPlatform::discover(vm, config); - let fdt = crate::boot::fdt::loongarch64::guest_firmware_dtb::build(&platform)?; + let fdt = fdt::guest_firmware_dtb::build(&platform)?; debug!( "VM[{}] loading LoongArch UEFI firmware FDT: {} bytes at {:#x}", config.base.id, @@ -187,6 +218,128 @@ pub fn emulated_fw_cfg(config: &AxVMCrateConfig) -> AxResult<&axvmconfig::Emulat .ok_or_else(|| ax_err_type!(NotFound, "LoongArch UEFI boot requires a fw_cfg device")) } +impl BootImagePlatform for super::LoongArch64Arch { + fn load_images_from_memory( + loader: &mut ImageLoaderCore<'_>, + images: StaticVmImage, + ) -> AxResult { + ensure_uefi_boot(loader)?; + load_uefi_firmware_dtb(loader)?; + add_uefi_fw_cfg(loader, images.kernel, images.ramdisk)?; + let firmware = images + .bios + .or_else(|| provider_firmware_image(loader)) + .ok_or_else(|| { + ax_err_type!( + NotFound, + "LoongArch UEFI boot requires a build-time firmware image" + ) + })?; + load_uefi_firmware_image(loader, firmware) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + fn load_images_from_filesystem(loader: &mut ImageLoaderCore<'_>) -> AxResult { + ensure_uefi_boot(loader)?; + load_uefi_firmware_dtb(loader)?; + + let kernel = crate::boot::images::fs::read_full_image( + &loader.config.kernel.kernel_path, + loader.provider, + )?; + let kernel: &'static [u8] = Box::leak(kernel.into_boxed_slice()); + let ramdisk = if let Some(path) = &loader.config.kernel.ramdisk_path { + let ramdisk = crate::boot::images::fs::read_full_image(path, loader.provider)?; + Some(Box::leak(ramdisk.into_boxed_slice()) as &'static [u8]) + } else { + None + }; + add_uefi_fw_cfg(loader, kernel, ramdisk)?; + + let firmware = provider_firmware_image(loader).ok_or_else(|| { + ax_err_type!( + NotFound, + "LoongArch UEFI boot requires a build-time firmware image" + ) + })?; + load_uefi_firmware_image(loader, firmware) + } +} + +fn ensure_uefi_boot(loader: &ImageLoaderCore<'_>) -> AxResult { + if loader.config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { + Ok(()) + } else { + ax_errno::ax_err!(Unsupported, "LoongArch guests require UEFI boot") + } +} + +fn load_uefi_firmware_dtb(loader: &ImageLoaderCore<'_>) -> AxResult { + prepare_uefi_runtime_config(&loader.vm, &loader.config); + load_firmware_fdt(&loader.vm, &loader.config) +} + +fn add_uefi_fw_cfg( + loader: &ImageLoaderCore<'_>, + kernel: &'static [u8], + ramdisk: Option<&'static [u8]>, +) -> AxResult { + let fw_cfg = emulated_fw_cfg(&loader.config)?; + loader.vm.add_fw_cfg_device(crate::FwCfgDeviceConfig { + base: GuestPhysAddr::from(fw_cfg.base_gpa), + size: fw_cfg.length, + kernel, + initrd: ramdisk, + cmdline: loader.config.kernel.cmdline.clone(), + cpu_num: loader.config.base.cpu_num as u16, + platform: fw_cfg_platform_config(&loader.vm, &loader.config), + }) +} + +fn provider_firmware_image(loader: &ImageLoaderCore<'_>) -> Option<&'static [u8]> { + loader + .provider + .static_firmware_images() + .iter() + .find(|image| image.id == loader.config.base.id) + .and_then(|image| image.bios) +} + +fn load_uefi_firmware_image(loader: &ImageLoaderCore<'_>, firmware: &[u8]) -> AxResult { + let load_gpa = loader + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "LoongArch UEFI firmware load addr is missed"))?; + let flash_len = loader + .config + .kernel + .memory_regions + .iter() + .find(|region| region.gpa == load_gpa.as_usize()) + .map_or(firmware.len(), |region| region.size); + fill_vm_region(load_gpa, flash_len, 0xff, loader.vm.clone())?; + load_vm_image_from_memory(firmware, load_gpa, loader.vm.clone()) +} + +fn fill_vm_region(load_addr: GuestPhysAddr, size: usize, byte: u8, vm: AxVMRef) -> AxResult { + let regions = vm.get_image_load_region(load_addr, size)?; + let mut filled_size = 0; + for region in regions { + // SAFETY: AxVM returned this writable guest-memory region and the fill + // is bounded by its length. + unsafe { core::ptr::write_bytes(region.as_mut_ptr(), byte, region.len()) }; + crate::clean_dcache_range((region.as_ptr() as usize).into(), region.len()); + filled_size += region.len(); + } + if filled_size == size { + Ok(()) + } else { + ax_errno::ax_err!( + InvalidData, + format!("VM memory was only partially filled: {filled_size}/{size} bytes") + ) + } +} + fn ram_regions(vm: &AxVMRef) -> Vec { let mut regions = vm .memory_regions() diff --git a/virtualization/axvm/src/boot/guest_platform/loongarch64/probe.rs b/virtualization/axvm/src/arch/loongarch64/boot/probe.rs similarity index 100% rename from virtualization/axvm/src/boot/guest_platform/loongarch64/probe.rs rename to virtualization/axvm/src/arch/loongarch64/boot/probe.rs diff --git a/virtualization/axvm/src/boot/guest_platform/loongarch64/resources.rs b/virtualization/axvm/src/arch/loongarch64/boot/resources.rs similarity index 100% rename from virtualization/axvm/src/boot/guest_platform/loongarch64/resources.rs rename to virtualization/axvm/src/arch/loongarch64/boot/resources.rs diff --git a/virtualization/axvm/src/arch/loongarch64/capabilities.rs b/virtualization/axvm/src/arch/loongarch64/capabilities.rs new file mode 100644 index 0000000000..cba7b09970 --- /dev/null +++ b/virtualization/axvm/src/arch/loongarch64/capabilities.rs @@ -0,0 +1,43 @@ +//! LoongArch64 implementations of AxVM platform capability hooks. + +use super::LoongArch64Arch; +use crate::architecture::{GuestBootPlatform, HostTimePlatform}; + +impl GuestBootPlatform for LoongArch64Arch { + fn init_guest_boot_resources() { + super::boot::init(); + } + + fn prepare_guest_boot( + vm_config: &mut crate::config::AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + _provider: &dyn crate::boot::BootImageProvider, + ) -> ax_errno::AxResult> { + if vm_create_config.kernel.effective_boot_protocol() != axvmconfig::VMBootProtocol::Uefi { + return ax_errno::ax_err!( + Unsupported, + "LoongArch AxVisor guests currently require UEFI boot" + ); + } + super::boot::prepare_uefi_fdt_config(vm_config, vm_create_config)?; + Ok(None) + } +} + +impl HostTimePlatform for LoongArch64Arch { + fn set_oneshot_timer(_deadline_ns: u64) {} + + fn register_timer_callback() { + ax_std::os::arceos::modules::ax_task::register_timer_callback(|_| { + crate::check_timer_events(); + }); + } +} + +pub fn host_fdt_bootarg() -> usize { + ax_std::os::arceos::modules::ax_hal::dtb::get_bootarg() +} + +pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + ax_std::os::arceos::modules::ax_hal::mem::phys_to_virt(paddr) +} diff --git a/virtualization/axvm/src/arch/loongarch64/fdt.rs b/virtualization/axvm/src/arch/loongarch64/fdt.rs new file mode 100644 index 0000000000..eca32a8fd2 --- /dev/null +++ b/virtualization/axvm/src/arch/loongarch64/fdt.rs @@ -0,0 +1,24 @@ +//! LoongArch compatibility facade for UEFI guest FDT preparation. + +use ax_errno::AxResult; +use axvmconfig::{AxVMCrateConfig, VMBootProtocol}; + +use crate::config::AxVMConfig; + +pub fn init_guest_boot_resources() { + super::boot::init(); +} + +pub fn handle_fdt_operations( + vm_config: &mut AxVMConfig, + vm_create_config: &mut AxVMCrateConfig, +) -> AxResult { + if vm_create_config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { + return super::boot::prepare_uefi_fdt_config(vm_config, vm_create_config); + } + + ax_errno::ax_err!( + Unsupported, + "LoongArch AxVisor guests currently require UEFI boot" + ) +} diff --git a/virtualization/axvm/src/arch/loongarch64/idle.rs b/virtualization/axvm/src/arch/loongarch64/idle.rs new file mode 100644 index 0000000000..4b97b55d50 --- /dev/null +++ b/virtualization/axvm/src/arch/loongarch64/idle.rs @@ -0,0 +1,26 @@ +//! LoongArch guest idle handling while host interrupts remain available. + +use super::AxvmLoongArchVcpu; + +pub(crate) fn wait(vcpu: &crate::vm::AxVCpuRef) { + crate::check_timer_events(); + if vcpu.get_arch_vcpu().has_enabled_pending_interrupt() { + trace!( + "VM[{}] VCpu[{}] skips idle wait because guest has enabled pending interrupt", + vcpu.vm_id(), + vcpu.id() + ); + return; + } + let idle_timeout = vcpu.get_arch_vcpu().idle_wait_timeout(); + trace!( + "VM[{}] VCpu[{}] host idle wait for {idle_timeout:?}", + vcpu.vm_id(), + vcpu.id() + ); + ax_std::os::arceos::modules::ax_hal::asm::set_timer_irq_enabled(true); + ax_std::os::arceos::modules::ax_hal::asm::enable_irqs(); + ax_std::os::arceos::modules::ax_hal::time::busy_wait(idle_timeout); + ax_std::os::arceos::modules::ax_hal::asm::disable_irqs(); + ax_std::os::arceos::modules::ax_hal::asm::set_timer_irq_enabled(false); +} diff --git a/virtualization/axvm/src/runtime/loongarch_irq.rs b/virtualization/axvm/src/arch/loongarch64/irq.rs similarity index 59% rename from virtualization/axvm/src/runtime/loongarch_irq.rs rename to virtualization/axvm/src/arch/loongarch64/irq.rs index 2040e974e8..3a205f3923 100644 --- a/virtualization/axvm/src/runtime/loongarch_irq.rs +++ b/virtualization/axvm/src/arch/loongarch64/irq.rs @@ -5,7 +5,7 @@ const EIOINTC_IRQ: usize = 3; /// Register the platform IRQ injector for LoongArch dynamic hypervisor builds. pub(crate) fn register_platform_irq_injector() { ax_plat::irq::loongarch64_hv::register_virtual_irq_injector(inject_platform_irq); - crate::host::arceos::set_irq_enabled(EIOINTC_IRQ, true); + set_irq_enabled(EIOINTC_IRQ, true); } /// Route a host physical IRQ to a LoongArch guest interrupt vector. @@ -28,6 +28,31 @@ pub fn unregister_guest_irq_routes(vm_id: usize) { ax_plat::irq::loongarch64_hv::unregister_guest_irq_routes(vm_id); } +fn set_irq_enabled(raw_irq: usize, enabled: bool) { + use ax_std::os::arceos::modules::ax_hal::irq::{self, IrqSource}; + + let gsi = match u32::try_from(raw_irq) { + Ok(gsi) => gsi, + Err(_) => { + warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: out of GSI range"); + return; + } + }; + let irq = match irq::resolve_irq_source(IrqSource::AcpiGsi(gsi)) { + Ok(irq) => irq, + Err(err) => { + warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: {err:?}"); + return; + } + }; + if let Err(err) = irq::set_enable(irq, enabled) { + warn!( + "failed to set LoongArch passthrough IRQ {raw_irq} ({irq:?}) enabled={enabled}: \ + {err:?}" + ); + } +} + fn inject_platform_irq(vm_id: usize, vcpu_id: usize, vector: usize, physical_irq: usize) { if let Err(err) = crate::runtime::vcpus::queue_external_interrupt(vm_id, vcpu_id, vector, physical_irq) diff --git a/virtualization/axvm/src/arch/loongarch64/mod.rs b/virtualization/axvm/src/arch/loongarch64/mod.rs index bbec7f0bf7..b42a41f2c6 100644 --- a/virtualization/axvm/src/arch/loongarch64/mod.rs +++ b/virtualization/axvm/src/arch/loongarch64/mod.rs @@ -9,19 +9,23 @@ use axvm_types::{ }; use loongarch_vcpu::{ LoongArchAccessFlags, LoongArchAccessWidth, LoongArchGuestPhysAddr, LoongArchHostOps, - LoongArchHostPhysAddr, LoongArchHostVirtAddr, LoongArchIocsrStateRef, - LoongArchNestedPagingConfig, LoongArchPerCpu, LoongArchVCpuCreateConfig, - LoongArchVCpuSetupConfig, LoongArchVcpu, LoongArchVcpuError, LoongArchVcpuResult, - LoongArchVmExit, + LoongArchHostPhysAddr, LoongArchHostVirtAddr, LoongArchNestedPagingConfig, LoongArchPerCpu, + LoongArchVCpuCreateConfig, LoongArchVCpuSetupConfig, LoongArchVcpu, LoongArchVcpuError, + LoongArchVcpuResult, LoongArchVmExit, }; -use super::{ - ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuCreateContext, - VcpuRunAction, VcpuSetupContext, -}; +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; use crate::host::{HostMemory, HostTime, default_host}; +pub(crate) mod boot; +mod capabilities; +pub(crate) mod fdt; +mod idle; +pub(crate) mod irq; mod npt; +mod vm; + +pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; pub(crate) struct LoongArch64Arch; @@ -34,7 +38,6 @@ pub(crate) enum LoongArchDeferredRunWork { impl ArchOps for LoongArch64Arch { type VCpu = AxvmLoongArchVcpu; type PerCpu = AxvmLoongArchPerCpu; - type VcpuCreateState = LoongArchIocsrStateRef; type DeferredRunWork = LoongArchDeferredRunWork; type NestedPageTable = npt::NestedPageTable; @@ -42,50 +45,8 @@ impl ArchOps for LoongArch64Arch { loongarch_vcpu::has_hardware_support() } - fn new_vcpu_create_state( - vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - let vcpu_state_count = vcpu_mappings - .iter() - .map(|(vcpu_id, ..)| *vcpu_id) - .max() - .map_or(0, |vcpu_id| vcpu_id + 1); - loongarch_result(loongarch_vcpu::LoongArchIocsrState::new(vcpu_state_count)) - } - - fn build_vcpu_create_config( - state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig> { - Ok(LoongArchVCpuCreateConfig { - cpu_id: ctx.vcpu_id, - dtb_addr: ctx.dtb_addr.unwrap_or_default().as_usize(), - boot_args: [0; 3], - boot_stack_top: 0, - firmware_boot: ctx.firmware_boot, - iocsr_state: state.clone(), - }) - } - - fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, - ) -> AxResult<::SetupConfig> { - let passthrough = ctx.interrupt_mode == axvm_types::VMInterruptMode::Passthrough; - Ok(LoongArchVCpuSetupConfig { - passthrough_interrupt: passthrough, - passthrough_timer: passthrough, - boot_args: [0; 3], - boot_stack_top: 0, - firmware_boot: ctx.firmware_boot, - }) - } - - fn new_nested_page_table(levels: usize) -> AxResult { - npt::NestedPageTable::new(levels) - } - fn register_platform_irq_injector() { - crate::runtime::loongarch_irq::register_platform_irq_injector(); + irq::register_platform_irq_injector(); } fn inject_pending_interrupt( @@ -146,29 +107,6 @@ impl ArchOps for LoongArch64Arch { drain_loongarch_pch_pic_events(vm); } - fn handle_idle(_vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { - crate::check_timer_events(); - if vcpu.get_arch_vcpu().has_enabled_pending_interrupt() { - trace!( - "VM[{}] VCpu[{}] skips idle wait because guest has enabled pending interrupt", - vcpu.vm_id(), - vcpu.id() - ); - return; - } - let idle_timeout = vcpu.get_arch_vcpu().idle_wait_timeout(); - trace!( - "VM[{}] VCpu[{}] host idle wait for {idle_timeout:?}", - vcpu.vm_id(), - vcpu.id() - ); - ax_std::os::arceos::modules::ax_hal::asm::set_timer_irq_enabled(true); - ax_std::os::arceos::modules::ax_hal::asm::enable_irqs(); - ax_std::os::arceos::modules::ax_hal::time::busy_wait(idle_timeout); - ax_std::os::arceos::modules::ax_hal::asm::disable_irqs(); - ax_std::os::arceos::modules::ax_hal::asm::set_timer_irq_enabled(false); - } - fn handle_vcpu_exit_bound( vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef, @@ -220,9 +158,15 @@ impl ArchOps for LoongArch64Arch { } LoongArchVmExit::Halt => { debug!("VM[{}] run VCpu[{}] Halt", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(Self::handle_halt())) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: true, + stop_reason: None, + })) } - LoongArchVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)), + LoongArchVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })), _ => Err(AxError::Unsupported), } } @@ -236,9 +180,12 @@ impl ArchOps for LoongArch64Arch { LoongArchDeferredRunWork::ExternalInterrupt { vector } => { Self::after_external_interrupt(vm, vcpu, vector); } - LoongArchDeferredRunWork::Idle => Self::handle_idle(vm, vcpu), + LoongArchDeferredRunWork::Idle => idle::wait(vcpu), } - Ok(VcpuRunAction::Yield) + Ok(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }) } fn clean_dcache_range(addr: VirtAddr, size: usize) { @@ -264,7 +211,10 @@ fn handle_loongarch_nested_page_fault( vcpu.id(), ax_addr.as_usize() ); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); + return Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })); }; return LoongArch64Arch::handle_vcpu_exit_bound(vm, vcpu, decoded); } @@ -280,7 +230,10 @@ fn handle_loongarch_nested_page_fault( ax_addr.as_usize(), ax_flags ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } } @@ -343,11 +296,11 @@ impl LoongArchHostOps for AxvmLoongArchHostOps { deadline: Duration, callback: Box, ) -> usize { - default_host().register_timer(deadline.as_nanos() as u64, callback) + crate::timer::register_timer(deadline.as_nanos() as u64, callback) } fn cancel_timer(token: usize) { - default_host().cancel_timer(token); + crate::timer::cancel_timer(token); } fn inject_interrupt(vm_id: usize, vcpu_id: usize, vector: usize) { diff --git a/virtualization/axvm/src/arch/loongarch64/npt.rs b/virtualization/axvm/src/arch/loongarch64/npt.rs index 6eaa824ff8..876d1a865a 100644 --- a/virtualization/axvm/src/arch/loongarch64/npt.rs +++ b/virtualization/axvm/src/arch/loongarch64/npt.rs @@ -197,12 +197,8 @@ impl ptg::TableMeta for LoongArchPagingMetaDataL4 { } } -pub(crate) type NestedPageTable = crate::arch::npt::LeveledPageTable< - LoongArchPagingMetaDataL3, - LoongArchPagingMetaDataL4, - H, - true, ->; +pub(crate) type NestedPageTable = + crate::npt::LeveledPageTable; fn config_to_flags(config: ptg::PteConfig) -> MappingFlags { let mut flags = MappingFlags::empty(); diff --git a/virtualization/axvm/src/arch/loongarch64/vm.rs b/virtualization/axvm/src/arch/loongarch64/vm.rs new file mode 100644 index 0000000000..0c78c57974 --- /dev/null +++ b/virtualization/axvm/src/arch/loongarch64/vm.rs @@ -0,0 +1,128 @@ +//! LoongArch64 VM resource creation and initialization. + +use ax_errno::AxResult; +use axvm_types::{NestedPagingConfig, VmArchVcpuOps}; +use loongarch_vcpu::{LoongArchVCpuCreateConfig, LoongArchVCpuSetupConfig}; + +use super::{LoongArch64Arch, loongarch_result, npt}; +use crate::{ + config::AxVMConfig, + vm::{ + AxVM, AxVMResources, + prepare::{ + PreparedVm, VmInitRequest, + address_space::{guest_owned_regions, map_guest_address_space}, + complete_vm_init, default_device_factories, + devices::PreparedDevices, + validate_guest_dtb, + vcpus::{PreparedVcpus, vcpu_placements}, + }, + }, +}; + +impl LoongArch64Arch { + pub(crate) fn create_vm_resources(config: AxVMConfig) -> AxResult { + let placements = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); + let levels = guest_page_table_levels(&placements); + let page_table = npt::NestedPageTable::new(levels)?; + AxVMResources::from_page_table(config, page_table, |root_paddr| { + let gpa_bits = match levels { + 3 => 39, + 4 => 48, + _ => { + return ax_errno::ax_err!( + InvalidInput, + "unsupported LoongArch nested page-table levels" + ); + } + }; + Ok(NestedPagingConfig::new(root_paddr, levels, gpa_bits, 0)) + }) + } + + pub(crate) fn init_vm(vm: &AxVM, request: VmInitRequest<'_>) -> AxResult { + match request { + VmInitRequest::Default => { + let factories = default_device_factories()?; + let interrupt_fabric = crate::InterruptFabric::new(vm.interrupt_mode()); + init_vm_with(vm, &factories, interrupt_fabric) + } + VmInitRequest::Provided { + factories, + interrupt_fabric, + } => init_vm_with(vm, factories, interrupt_fabric), + } + } +} + +fn init_vm_with( + vm: &AxVM, + factories: &axdevice::DeviceFactoryRegistry, + interrupt_fabric: crate::InterruptFabric, +) -> AxResult { + complete_vm_init(vm, interrupt_fabric, |resources, interrupt_fabric| { + let placements = vcpu_placements(resources); + let state_count = placements + .iter() + .map(|placement| placement.id) + .max() + .map_or(0, |vcpu_id| vcpu_id + 1); + let iocsr_state = loongarch_result(loongarch_vcpu::LoongArchIocsrState::new(state_count))?; + let dtb_addr = resources + .config() + .image_config() + .dtb_load_gpa + .unwrap_or_default(); + let firmware_boot = uses_firmware_boot(resources.config()); + let vcpus = PreparedVcpus::create(vm.id(), &placements, |placement| { + Ok(LoongArchVCpuCreateConfig { + cpu_id: placement.id, + dtb_addr: dtb_addr.as_usize(), + boot_args: [0; 3], + boot_stack_top: 0, + firmware_boot, + iocsr_state: iocsr_state.clone(), + }) + })?; + let mut devices = PreparedDevices::build_common(resources, factories, interrupt_fabric)?; + devices.register_special_devices(vm)?; + validate_guest_dtb(resources)?; + + let owned_regions = guest_owned_regions(resources); + map_guest_address_space(vm, resources, devices.devices(), &owned_regions)?; + vcpus.setup(resources, build_vcpu_setup_config)?; + + Ok(PreparedVm::new(vcpus, devices)) + }) +} + +fn build_vcpu_setup_config( + config: &AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], +) -> AxResult<::SetupConfig> { + let passthrough = config.interrupt_mode() == axvm_types::VMInterruptMode::Passthrough; + Ok(LoongArchVCpuSetupConfig { + passthrough_interrupt: passthrough, + passthrough_timer: passthrough, + boot_args: [0; 3], + boot_stack_top: 0, + firmware_boot: uses_firmware_boot(config), + }) +} + +fn uses_firmware_boot(config: &AxVMConfig) -> bool { + matches!( + config.boot_policy(), + crate::config::GuestBootPolicy::AdjustKernelForBootProtocol { + protocol: crate::config::VMBootProtocol::Uefi, + } + ) +} + +fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> usize { + let mut levels = 4; + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + levels = levels.min(crate::percpu::cpu_max_guest_page_table_levels(cpu_id).unwrap_or(4)); + } + levels +} diff --git a/virtualization/axvm/src/arch/mod.rs b/virtualization/axvm/src/arch/mod.rs index eff1db411f..c1db6f0739 100644 --- a/virtualization/axvm/src/arch/mod.rs +++ b/virtualization/axvm/src/arch/mod.rs @@ -1,698 +1,124 @@ -//! Architecture component glue owned by AxVM. +//! Target architecture selection and stable internal dispatch. -use alloc::{format, vec::Vec}; +use ax_errno::AxResult; -use ax_errno::{AxResult, ax_err}; -use ax_memory_addr::{PhysAddr, VirtAddr}; -use axaddrspace::NestedPageTableOps; -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -use axvm_types::SysRegAddr; -use axvm_types::{ - AccessWidth, GuestPhysAddr, NestedPagingConfig, PassThroughPortConfig, VMInterruptMode, - VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState, -}; -#[cfg(target_arch = "x86_64")] -use axvm_types::{MappingFlags, Port}; - -#[cfg(target_arch = "aarch64")] -use crate::CpuMask; -use crate::StopReason; +pub(crate) use crate::architecture::*; +use crate::architecture::{BootImagePlatform, GuestBootPlatform, HostTimePlatform}; #[cfg(target_arch = "aarch64")] mod aarch64; #[cfg(target_arch = "loongarch64")] mod loongarch64; -mod npt; #[cfg(target_arch = "riscv64")] mod riscv64; #[cfg(target_arch = "x86_64")] mod x86_64; #[cfg(target_arch = "aarch64")] -pub(crate) type CurrentArch = aarch64::Aarch64Arch; +pub(crate) use aarch64::Aarch64Arch as CurrentArch; +#[cfg(target_arch = "aarch64")] +pub use aarch64::ImageLoader; +#[cfg(target_arch = "aarch64")] +pub(crate) use aarch64::fdt; #[cfg(target_arch = "loongarch64")] -pub(crate) type CurrentArch = loongarch64::LoongArch64Arch; -#[cfg(target_arch = "riscv64")] -pub(crate) type CurrentArch = riscv64::Riscv64Arch; -#[cfg(target_arch = "x86_64")] -pub(crate) type CurrentArch = x86_64::X86_64Arch; - -pub(crate) type ArchVCpu = ::VCpu; -pub(crate) type ArchPerCpu = ::PerCpu; -pub(crate) type ArchNestedPageTable = ::NestedPageTable; - -pub(crate) fn guest_page_table_levels( - vcpu_mappings: &[(usize, Option, usize)], -) -> AxResult { - CurrentArch::guest_page_table_levels(vcpu_mappings) -} - -pub(crate) fn new_nested_page_table(levels: usize) -> AxResult { - CurrentArch::new_nested_page_table(levels) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn register_x86_arch_device( - config: &axvm_types::EmulatedDeviceConfig, - devices: &mut axdevice::AxVmDevices, -) -> AxResult { - x86_64::register_arch_device(config, devices) -} - -#[cfg(all(target_arch = "x86_64", feature = "vmx"))] -pub(crate) fn x86_apic_access_page_addr() -> axvm_types::HostPhysAddr { - x86_64::x86_apic_access_page_addr() -} - -pub(crate) fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - vcpu_mappings: &[(usize, Option, usize)], -) -> AxResult { - CurrentArch::nested_paging_config(root_paddr, levels, vcpu_mappings) -} - -/// Runtime scheduler action selected after an architecture-local vCPU exit. -#[derive(Debug)] -pub(crate) enum VcpuRunAction { - /// Return to the runtime loop without blocking. - Yield, - /// Block the current vCPU task on the VM runtime wait queue. - Wait, - /// Request VM stop with the provided reason. - Stop(StopReason), -} - -/// Result of handling one exit while the vCPU is still bound to the host CPU. -#[derive(Debug)] -pub(crate) enum BoundVcpuExit { - /// The exit was handled completely; re-enter the guest in the current run slice. - Continue, - /// The run slice is complete and can return this scheduler action after unbind. - Complete(VcpuRunAction), - /// Finish architecture-local work after unbinding the vCPU. - Defer(D), -} - -#[cfg(target_arch = "x86_64")] -#[derive(Clone, Copy, Debug)] -pub(crate) enum LegacyDeferredRunWork { - ExternalInterrupt { vector: usize }, - PreemptionTimer, - InterruptEnd { vector: Option }, -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct MmioReadExit { - pub(crate) addr: GuestPhysAddr, - pub(crate) width: AccessWidth, - pub(crate) reg: usize, - pub(crate) reg_width: AccessWidth, - pub(crate) signed_ext: bool, -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct MmioWriteExit { - pub(crate) addr: GuestPhysAddr, - pub(crate) width: AccessWidth, - pub(crate) data: u64, +pub(crate) use loongarch64::LoongArch64Arch as CurrentArch; +#[cfg(target_arch = "loongarch64")] +pub(crate) use loongarch64::boot as guest_platform; +#[cfg(target_arch = "loongarch64")] +pub use loongarch64::boot::ImageLoader; +#[cfg(target_arch = "loongarch64")] +pub(crate) use loongarch64::fdt; +#[cfg(not(target_arch = "loongarch64"))] +pub(crate) mod guest_platform { + #[doc(hidden)] + pub const SUPPORTED: bool = false; } - -#[derive(Clone, Copy, Debug)] +#[cfg(target_arch = "riscv64")] +pub use riscv64::ImageLoader; +#[cfg(target_arch = "riscv64")] +pub(crate) use riscv64::Riscv64Arch as CurrentArch; +#[cfg(target_arch = "riscv64")] +pub(crate) use riscv64::fdt; #[cfg(target_arch = "x86_64")] -pub(crate) struct IoReadExit { - pub(crate) port: Port, - pub(crate) width: AccessWidth, -} - -#[derive(Clone, Copy, Debug)] +pub(crate) use x86_64::X86_64Arch as CurrentArch; #[cfg(target_arch = "x86_64")] -pub(crate) struct IoWriteExit { - pub(crate) port: Port, - pub(crate) width: AccessWidth, - pub(crate) data: u64, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub(crate) struct SysRegReadExit { - pub(crate) addr: SysRegAddr, - pub(crate) reg: usize, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub(crate) struct SysRegWriteExit { - pub(crate) addr: SysRegAddr, - pub(crate) value: u64, -} - -#[derive(Clone, Copy, Debug)] +pub use x86_64::boot::ImageLoader; #[cfg(target_arch = "x86_64")] -pub(crate) struct NestedPageFaultExit { - pub(crate) addr: GuestPhysAddr, - pub(crate) access_flags: MappingFlags, -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct HypercallExit { - pub(crate) nr: u64, - pub(crate) args: [u64; 6], -} - -#[derive(Clone, Copy, Debug)] -#[cfg(not(target_arch = "x86_64"))] -pub(crate) struct CpuUpExit { - pub(crate) target_cpu: u64, - pub(crate) entry_point: GuestPhysAddr, - pub(crate) arg: u64, -} - -#[derive(Clone, Copy, Debug)] -#[cfg(target_arch = "aarch64")] -pub(crate) struct SendIpiExit { - pub(crate) target_cpu: u64, - pub(crate) target_cpu_aux: u64, - pub(crate) send_to_all: bool, - pub(crate) send_to_self: bool, - pub(crate) vector: u64, -} - -#[allow(dead_code)] -pub(crate) struct VcpuCreateContext { - pub(crate) vcpu_id: usize, - pub(crate) phys_cpu_id: usize, - pub(crate) dtb_addr: Option, - pub(crate) firmware_boot: bool, -} - -#[allow(dead_code)] -pub(crate) struct VcpuSetupContext<'a> { - pub(crate) interrupt_mode: VMInterruptMode, - pub(crate) emulates_console: bool, - pub(crate) passthrough_ports: &'a [PassThroughPortConfig], - pub(crate) memory_regions: &'a [crate::vm::VMMemoryRegion], - pub(crate) firmware_boot: bool, -} - -pub(crate) trait ArchOps { - type VCpu: VmArchVcpuOps; - type PerCpu: VmArchPerCpuOps; - type VcpuCreateState; - type DeferredRunWork; - type NestedPageTable: NestedPageTableOps; - - fn has_hardware_support() -> bool; - - fn max_guest_page_table_levels() -> usize { - 4 - } - - fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { - let mut levels = Self::max_guest_page_table_levels(); - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - levels = levels.min( - crate::percpu::cpu_max_guest_page_table_levels(cpu_id) - .unwrap_or_else(Self::max_guest_page_table_levels), - ); - } - Ok(levels) - } - - fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - let gpa_bits = match levels { - 3 => 39, - 4 => 48, - _ => return ax_errno::ax_err!(InvalidInput, "unsupported nested page-table levels"), - }; - Ok(NestedPagingConfig::new(root_paddr, levels, gpa_bits, 0)) - } - - fn new_nested_page_table(levels: usize) -> AxResult; - - fn clean_dcache_range(_addr: VirtAddr, _size: usize) {} - - fn new_vcpu_create_state( - vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult; - - fn build_vcpu_create_config( - state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig>; - - fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, - ) -> AxResult<::SetupConfig>; - - fn register_platform_irq_injector() {} - - fn vcpu_affinities( - cpu_num: usize, - phys_cpu_ids: Option<&[usize]>, - phys_cpu_sets: Option<&[usize]>, - ) -> Vec<(usize, Option, usize)> { - default_vcpu_affinities(cpu_num, phys_cpu_ids, phys_cpu_sets) - } +pub(crate) use x86_64::fdt; +/// Architecture-specific public compatibility exports. +pub mod platform { #[cfg(target_arch = "aarch64")] - fn ipi_targets( - vm: &crate::AxVMRef, - current_vcpu_id: usize, - target_cpu: u64, - target_cpu_aux: u64, - send_to_all: bool, - send_to_self: bool, - ) -> CpuMask<64> { - let mut targets = CpuMask::new(); - - if send_to_all { - for vcpu in vm.vcpu_list() { - if vcpu.id() != current_vcpu_id { - targets.set(vcpu.id(), true); - } - } - } else if send_to_self { - targets.set(current_vcpu_id, true); - } else { - let _ = target_cpu_aux; - targets.set(target_cpu as usize, true); - } - - targets - } - - #[cfg(not(target_arch = "x86_64"))] - fn set_vcpu_on_args(vcpu: &crate::vm::AxVCpuRef, _vcpu_id: usize, arg: usize) { - vcpu.set_gpr(0, arg); - } - - #[cfg(not(target_arch = "x86_64"))] - fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { - vcpu.set_gpr(0, 0); - } - - #[cfg(target_arch = "x86_64")] - fn set_io_read_result(vcpu: &crate::vm::AxVCpuRef, val: usize) { - vcpu.set_gpr(0, val); - } - - fn before_first_run(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) {} - - fn before_vcpu_run(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) {} - - fn inject_pending_interrupt( - _vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - interrupt: crate::vm::PendingInterrupt, - ) { - match interrupt { - crate::vm::PendingInterrupt::Normal(vector) => { - trace!( - "Injecting queued interrupt {vector:#x} into VM[{}] VCpu[{}]", - vcpu.vm_id(), - vcpu.id() - ); - if let Err(err) = vcpu.inject_interrupt(vector) { - warn!( - "Failed to inject queued interrupt {vector:#x} into VM[{}] VCpu[{}]: \ - {err:?}", - vcpu.vm_id(), - vcpu.id() - ); - } - } - crate::vm::PendingInterrupt::External { - vector, - physical_irq, - } => { - warn!( - "VM[{}] VCpu[{}] dropped unsupported external interrupt vector={vector:#x}, \ - physical_irq={physical_irq:#x}", - vcpu.vm_id(), - vcpu.id() - ); - } - } - } - - fn after_external_interrupt( - _vm: &crate::AxVMRef, - _vcpu: &crate::vm::AxVCpuRef, - vector: usize, - ) { - crate::host::arceos::dispatch_host_irq(vector); - crate::check_timer_events(); - } - - #[cfg(target_arch = "x86_64")] - fn after_preemption_timer(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) { - crate::check_timer_events(); - } - - #[cfg(target_arch = "x86_64")] - fn after_interrupt_end( - _vm: &crate::AxVMRef, - _vcpu: &crate::vm::AxVCpuRef, - _vector: Option, - ) { - } - + pub use super::aarch64::{host_fdt_bootarg, host_phys_to_virt}; #[cfg(target_arch = "loongarch64")] - fn handle_idle(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) { - crate::check_timer_events(); - } - - fn on_last_vcpu_exit(_vm_id: usize) {} - - fn after_mmio_write(_vm: &crate::AxVMRef) {} - - #[cfg(not(target_arch = "x86_64"))] - fn cpu_up_target_vcpu_id(vm: &crate::AxVMRef, target_cpu: u64) -> Option { - vm.get_vcpu_affinities_pcpu_ids() - .iter() - .find_map(|(vcpu_id, _, phys_id)| (*phys_id == target_cpu as usize).then_some(*vcpu_id)) - } - - #[cfg(not(target_arch = "aarch64"))] - fn handle_halt() -> VcpuRunAction { - VcpuRunAction::Wait - } - - fn handle_vcpu_exit_bound( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - exit: ::Exit, - ) -> AxResult>; - - fn finish_deferred_run_work( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - work: Self::DeferredRunWork, - ) -> AxResult; - - fn run_vcpu( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - ) -> AxResult - where - Self: Sized, - { - let vm_id = vm.id(); - let vcpu_id = vcpu.id(); - - match vcpu.state() { - VmVcpuState::Free => vcpu.bind()?, - VmVcpuState::Ready => {} - state => { - return ax_err!( - BadState, - format!("VCpu state is not Free or Ready, but {state:?}") - ); - } - } - - let run_result = vcpu.with_current_cpu_set(|| -> AxResult<_> { - loop { - crate::runtime::vcpus::inject_pending_interrupts::(vm.id(), vcpu_id, vcpu); - - let exit = vcpu.run()?; - trace!("{exit:#x?}"); - match Self::handle_vcpu_exit_bound(vm, vcpu, exit)? { - BoundVcpuExit::Continue => continue, - action => break Ok(action), - } - } - }); - - let unbind_result = vcpu.unbind(); - match run_result { - Ok(BoundVcpuExit::Complete(action)) => { - unbind_result?; - Ok(action) - } - Ok(BoundVcpuExit::Defer(work)) => { - unbind_result?; - Self::finish_deferred_run_work(vm, vcpu, work) - } - Ok(BoundVcpuExit::Continue) => unreachable!("continued exits do not leave run loop"), - Err(err) => { - if let Err(unbind_err) = unbind_result { - warn!( - "VM[{vm_id}] VCpu[{vcpu_id}] unbind after run error failed: {unbind_err:?}" - ); - } - Err(err) - } - } - } -} - -pub(crate) fn handle_mmio_read( - vm: &crate::AxVM, - vcpu: &crate::vm::AxVCpuRef, - exit: MmioReadExit, -) -> AxResult> { - let raw = vm.get_devices()?.handle_mmio_read(exit.addr, exit.width)?; - let masked = raw & crate::vm::width_mask(exit.width); - let val = if exit.signed_ext { - crate::vm::sign_extend_value(masked, exit.width) - } else { - masked & crate::vm::width_mask(exit.reg_width) + pub use super::loongarch64::irq::{ + register_guest_irq_route as register_loongarch_guest_irq_route, + unregister_guest_irq_routes as unregister_loongarch_guest_irq_routes, }; - vcpu.set_gpr(exit.reg, val); - Ok(BoundVcpuExit::Continue) -} - -pub(crate) fn handle_mmio_write( - vm: &crate::AxVMRef, - exit: MmioWriteExit, -) -> AxResult> { - vm.handle_mmio_write(exit.addr, exit.width, exit.data as usize)?; - A::after_mmio_write(vm); - Ok(BoundVcpuExit::Continue) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn handle_io_read( - vm: &crate::AxVM, - vcpu: &crate::vm::AxVCpuRef, - exit: IoReadExit, -) -> AxResult> { - let val = vm.get_devices()?.handle_port_read(exit.port, exit.width)?; - A::set_io_read_result(vcpu, val); - Ok(BoundVcpuExit::Continue) + #[cfg(target_arch = "loongarch64")] + pub use super::loongarch64::{host_fdt_bootarg, host_phys_to_virt}; + #[cfg(target_arch = "riscv64")] + pub use super::riscv64::{host_fdt_bootarg, host_phys_to_virt}; + #[cfg(target_arch = "x86_64")] + pub use super::x86_64::irq::{ + register_ioapic_irq_forwarding_activator as register_x86_ioapic_irq_forwarding_activator, + register_ioapic_irq_forwarding_route as register_x86_ioapic_irq_forwarding_route, + register_ioapic_irq_forwarding_route_with_trigger as register_x86_ioapic_irq_forwarding_route_with_trigger, + }; + #[cfg(all( + any(target_arch = "x86_64", target_arch = "loongarch64"), + any(feature = "fs", feature = "host-fs") + ))] + pub use crate::host::arceos::shutdown_host_filesystems; } -#[cfg(target_arch = "x86_64")] -pub(crate) fn handle_io_write( - vm: &crate::AxVM, - exit: IoWriteExit, -) -> AxResult> { - vm.get_devices()? - .handle_port_write(exit.port, exit.width, exit.data as usize)?; - Ok(BoundVcpuExit::Continue) -} +pub(crate) type ArchVCpu = ::VCpu; +pub(crate) type ArchPerCpu = ::PerCpu; +pub(crate) type ArchNestedPageTable = ::NestedPageTable; -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub(crate) fn handle_sys_reg_read( - vm: &crate::AxVM, - vcpu: &crate::vm::AxVCpuRef, - exit: SysRegReadExit, -) -> AxResult> { - let val = vm.get_devices()?.handle_sys_reg_read( - exit.addr, - // System registers are currently modeled as fixed-width device registers. - AccessWidth::Qword, - )?; - vcpu.set_gpr(exit.reg, val); - Ok(BoundVcpuExit::Continue) +pub(crate) fn register_timer_callback() { + CurrentArch::register_timer_callback(); } -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub(crate) fn handle_sys_reg_write( - vm: &crate::AxVM, - exit: SysRegWriteExit, -) -> AxResult> { - vm.get_devices()? - .handle_sys_reg_write(exit.addr, AccessWidth::Qword, exit.value as usize)?; - Ok(BoundVcpuExit::Continue) +pub(crate) fn set_oneshot_timer(deadline_ns: u64) { + CurrentArch::set_oneshot_timer(deadline_ns); } -pub(crate) fn handle_hypercall( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - exit: HypercallExit, -) -> AxResult> { - debug!("Hypercall [{:#x}] args {:x?}", exit.nr, exit.args); - match crate::runtime::hvc::HyperCall::new(vm.clone(), exit.nr, exit.args) { - Ok(hypercall) => { - let ret_val = match hypercall.execute() { - Ok(ret_val) => ret_val as isize, - Err(err) => { - warn!("Hypercall [{:#x}] failed: {err:?}", exit.nr); - -1 - } - }; - vcpu.set_return_value(ret_val as usize); - } - Err(err) => { - warn!("Hypercall [{:#x}] failed: {err:?}", exit.nr); - } - } - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) +pub(crate) fn init_guest_boot_resources() { + CurrentArch::init_guest_boot_resources(); } -#[cfg(not(target_arch = "x86_64"))] -pub(crate) fn handle_cpu_up( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - exit: CpuUpExit, -) -> AxResult> { - let vm_id = vm.id(); - let vcpu_id = vcpu.id(); - info!( - "VM[{vm_id}]'s VCpu[{vcpu_id}] try to boot target_cpu [{}] entry_point={:x} arg={:#x}", - exit.target_cpu, exit.entry_point, exit.arg - ); - - let Some(target_vcpu_id) = A::cpu_up_target_vcpu_id(vm, exit.target_cpu) else { - warn!( - "VM[{vm_id}] cannot resolve architecture CPU target {} to a VM-local vCPU", - exit.target_cpu - ); - vcpu.set_return_value(usize::MAX); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); - }; - - match crate::runtime::vcpus::vcpu_on( - vm.clone(), - target_vcpu_id, - exit.entry_point, - exit.arg as _, - ) { - Ok(()) => A::set_cpu_up_success(vcpu), - Err(err) => { - warn!("Failed to boot VM[{vm_id}] VCpu[{target_vcpu_id}]: {err:?}"); - vcpu.set_return_value(usize::MAX); - } - } - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) +pub(crate) fn prepare_guest_boot( + vm_config: &mut crate::config::AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn crate::boot::BootImageProvider, +) -> AxResult> { + CurrentArch::prepare_guest_boot(vm_config, vm_create_config, provider) } -#[cfg(target_arch = "aarch64")] -pub(crate) fn handle_send_ipi( - vm: &crate::AxVMRef, - vcpu_id: usize, - exit: SendIpiExit, -) -> AxResult> { - let vm_id = vm.id(); - debug!( - "VM[{vm_id}] run VCpu[{vcpu_id}] SendIPI, target_cpu={:#x}, target_cpu_aux={:#x}, \ - vector={}", - exit.target_cpu, exit.target_cpu_aux, exit.vector - ); - let targets = A::ipi_targets( - vm, - vcpu_id, - exit.target_cpu, - exit.target_cpu_aux, - exit.send_to_all, - exit.send_to_self, - ); - if targets.is_empty() { - warn!( - "VM[{vm_id}] SendIPI has no target: target_cpu={:#x}, target_cpu_aux={:#x}", - exit.target_cpu, exit.target_cpu_aux - ); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); - } - - if targets.get(vcpu_id) { - crate::inject_current_vcpu_interrupt(exit.vector as _) - .expect("failed to inject self IPI into current vCPU"); - } - let mut remote_targets = targets; - remote_targets.set(vcpu_id, false); - if !remote_targets.is_empty() - && let Err(err) = vm.inject_interrupt_to_vcpu(remote_targets, exit.vector as _) - { - warn!( - "Failed to inject interrupt {} to VM[{vm_id}] targets {remote_targets:?}: {err:?}", - exit.vector - ); - } - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) +pub(crate) fn load_images_from_memory( + loader: &mut crate::boot::images::ImageLoaderCore<'_>, + images: crate::boot::StaticVmImage, +) -> AxResult { + CurrentArch::load_images_from_memory(loader, images) } -#[cfg(target_arch = "x86_64")] -pub(crate) fn finish_legacy_deferred_run_work( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - work: LegacyDeferredRunWork, -) -> AxResult -where - A: ArchOps, -{ - match work { - LegacyDeferredRunWork::ExternalInterrupt { vector } => { - A::after_external_interrupt(vm, vcpu, vector); - } - LegacyDeferredRunWork::PreemptionTimer => { - A::after_preemption_timer(vm, vcpu); - } - LegacyDeferredRunWork::InterruptEnd { vector } => { - A::after_interrupt_end(vm, vcpu, vector); - } - } - Ok(VcpuRunAction::Yield) +#[cfg(any(feature = "fs", feature = "host-fs"))] +pub(crate) fn load_images_from_filesystem( + loader: &mut crate::boot::images::ImageLoaderCore<'_>, +) -> AxResult { + CurrentArch::load_images_from_filesystem(loader) } -pub(crate) fn target_phys_cpu_ids(vcpu_mappings: &[(usize, Option, usize)]) -> Vec { - let mut cpu_ids = Vec::new(); - for (_, maybe_mask, phys_id) in vcpu_mappings { - if let Some(mask) = maybe_mask { - for cpu_id in 0..usize::BITS as usize { - if mask & (1usize << cpu_id) != 0 && !cpu_ids.contains(&cpu_id) { - cpu_ids.push(cpu_id); - } - } - } else if !cpu_ids.contains(phys_id) { - cpu_ids.push(*phys_id); - } - } - cpu_ids +pub(crate) fn is_x86_linux_image_config( + config: &axvmconfig::AxVMCrateConfig, + provider: &dyn crate::boot::BootImageProvider, +) -> bool { + CurrentArch::is_x86_linux_image_config(config, provider) } -pub(crate) fn default_vcpu_affinities( - cpu_num: usize, - phys_cpu_ids: Option<&[usize]>, - phys_cpu_sets: Option<&[usize]>, -) -> Vec<(usize, Option, usize)> { - let mut vcpus = Vec::with_capacity(cpu_num); - for vcpu_id in 0..cpu_num { - vcpus.push((vcpu_id, None, vcpu_id)); - } - - if let Some(phys_cpu_sets) = phys_cpu_sets { - for (vcpu_id, pcpu_mask_bitmap) in phys_cpu_sets.iter().enumerate() { - if let Some(vcpu) = vcpus.get_mut(vcpu_id) { - vcpu.1 = Some(*pcpu_mask_bitmap); - } - } - } - - if let Some(phys_cpu_ids) = phys_cpu_ids { - for (vcpu_id, phys_id) in phys_cpu_ids.iter().enumerate() { - if let Some(vcpu) = vcpus.get_mut(vcpu_id) { - vcpu.2 = *phys_id; - } - } - } - - vcpus +pub(crate) fn default_boot_firmware_load_gpa( + config: &axvmconfig::AxVMCrateConfig, +) -> Option { + CurrentArch::default_boot_firmware_load_gpa(config) } diff --git a/virtualization/axvm/src/arch/riscv64/capabilities.rs b/virtualization/axvm/src/arch/riscv64/capabilities.rs new file mode 100644 index 0000000000..70f2a1700e --- /dev/null +++ b/virtualization/axvm/src/arch/riscv64/capabilities.rs @@ -0,0 +1,86 @@ +//! RISC-V implementations of AxVM platform capability hooks. + +use alloc::{format, vec::Vec}; + +use ax_errno::{AxResult, ax_err_type}; + +use super::Riscv64Arch; +use crate::architecture::{BootImagePlatform, GuestBootPlatform, HostTimePlatform}; + +impl HostTimePlatform for Riscv64Arch {} + +impl BootImagePlatform for Riscv64Arch { + fn load_guest_dtb( + loader: &crate::boot::images::ImageLoaderCore<'_>, + dtb: &crate::boot::fdt::GuestDtbImage, + ) -> AxResult { + let bytes = dtb.as_bytes(); + let source = core::ptr::NonNull::new(bytes.as_ptr() as *mut u8) + .ok_or_else(|| ax_err_type!(InvalidData, "Guest DTB pointer is null"))?; + super::fdt::core::update_fdt(source, bytes.len(), loader.vm.clone(), &loader.config) + } +} + +impl GuestBootPlatform for Riscv64Arch { + fn prepare_guest_boot( + vm_config: &mut crate::config::AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn crate::boot::BootImageProvider, + ) -> AxResult> { + super::fdt::core::prepare_dtb_guest(vm_config, vm_create_config, provider) + } +} + +pub fn host_fdt_bootarg() -> usize { + ax_std::os::arceos::modules::ax_hal::dtb::get_bootarg() +} + +pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + ax_std::os::arceos::modules::ax_hal::mem::phys_to_virt(paddr) +} + +pub(super) fn decode_plic_source(specifier: &[u32]) -> Option { + specifier.first().copied().filter(|source| *source != 0) +} + +pub(super) fn patch_runtime_fdt( + fdt_bytes: &[u8], + vm: &crate::AxVMRef, + crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + let host_fdt = super::fdt::core::try_get_host_fdt() + .map(fdt_edit::Fdt::from_bytes) + .transpose() + .map_err(|err| { + ax_err_type!( + InvalidData, + format!("Failed to parse host FDT while updating guest FDT: {err:#?}") + ) + })?; + let guest_fdt = super::fdt::core::patch_guest_fdt_for_runtime( + fdt_bytes, + &vm.memory_regions(), + crate_config, + None, + false, + )?; + super::fdt::ensure_chosen_from_host(guest_fdt, host_fdt.as_ref()) +} + +pub(super) fn patch_provided_fdt( + provided_dtb: &[u8], + _host_dtb: Option<&[u8]>, + _crate_config: &axvmconfig::AxVMCrateConfig, +) -> AxResult> { + Ok(provided_dtb.to_vec()) +} + +#[cfg(test)] +mod tests { + #[test] + fn plic_interrupt_uses_first_nonzero_fdt_cell() { + assert_eq!(super::decode_plic_source(&[8]), Some(8)); + assert_eq!(super::decode_plic_source(&[0]), None); + assert_eq!(super::decode_plic_source(&[]), None); + } +} diff --git a/virtualization/axvm/src/arch/riscv64/fdt.rs b/virtualization/axvm/src/arch/riscv64/fdt.rs new file mode 100644 index 0000000000..8aa7a815b9 --- /dev/null +++ b/virtualization/axvm/src/arch/riscv64/fdt.rs @@ -0,0 +1,61 @@ +//! RISC-V compatibility facade and target-specific guest FDT policy. + +use alloc::vec::Vec; + +use ax_errno::AxResult; + +use crate::{ + boot::{BootImageProvider, fdt::GuestDtbImage}, + config::AxVMConfig, +}; + +#[path = "../../boot/fdt/core/mod.rs"] +pub(crate) mod core; + +pub use core::{ + parse_passthrough_devices_address, parse_reserved_memory_regions, parse_vm_interrupt, + reserve_excluded_device_ranges, set_phys_cpu_sets, setup_guest_fdt_from_vmm, try_get_host_fdt, + update_fdt, update_provided_fdt, +}; + +pub(crate) fn guest_fdt_policy() -> core::GuestFdtPolicy { + core::GuestFdtPolicy { + patch_runtime: super::capabilities::patch_runtime_fdt, + patch_provided: super::capabilities::patch_provided_fdt, + decode_interrupt: super::capabilities::decode_plic_source, + } +} + +pub(crate) fn host_fdt_bootarg() -> usize { + super::capabilities::host_fdt_bootarg() +} + +pub(crate) fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + super::capabilities::host_phys_to_virt(paddr) +} + +pub(super) fn ensure_chosen_from_host( + guest_dtb: Vec, + host_fdt: Option<&fdt_edit::Fdt>, +) -> AxResult> { + let Some(host_fdt) = host_fdt else { + return Ok(guest_dtb); + }; + let mut guest = core::tree::FdtTree::from_bytes(&guest_dtb)?; + if guest.inner().get_by_path_id("/chosen").is_some() { + return Ok(guest.finish()); + } + let Some(host_chosen) = host_fdt.get_by_path_id("/chosen") else { + return Ok(guest.finish()); + }; + guest.copy_subtree_from(host_fdt, host_chosen, guest.inner().root_id(), false)?; + Ok(guest.finish()) +} + +pub fn handle_fdt_operations( + vm_config: &mut AxVMConfig, + vm_create_config: &mut axvmconfig::AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult> { + core::prepare_dtb_guest(vm_config, vm_create_config, provider) +} diff --git a/virtualization/axvm/src/arch/riscv64/images.rs b/virtualization/axvm/src/arch/riscv64/images.rs new file mode 100644 index 0000000000..5a22968ad6 --- /dev/null +++ b/virtualization/axvm/src/arch/riscv64/images.rs @@ -0,0 +1,33 @@ +//! Public RISC-V image-loader facade preserving the DTB constructor contract. + +use ax_errno::AxResult; +use axvmconfig::AxVMCrateConfig; + +use crate::{ + AxVMRef, VMMemoryRegion, + boot::{BootImageProvider, fdt::GuestDtbImage, images::ImageLoaderCore}, +}; + +pub struct ImageLoader<'a>(ImageLoaderCore<'a>); + +impl<'a> ImageLoader<'a> { + pub fn new( + main_memory: VMMemoryRegion, + config: AxVMCrateConfig, + vm: AxVMRef, + provider: &'a dyn BootImageProvider, + guest_dtb: Option, + ) -> Self { + Self(ImageLoaderCore::new( + main_memory, + config, + vm, + provider, + guest_dtb, + )) + } + + pub fn load(&mut self) -> AxResult { + self.0.load() + } +} diff --git a/virtualization/axvm/src/irq/riscv.rs b/virtualization/axvm/src/arch/riscv64/irq.rs similarity index 99% rename from virtualization/axvm/src/irq/riscv.rs rename to virtualization/axvm/src/arch/riscv64/irq.rs index c514e9dfa8..af994514dc 100644 --- a/virtualization/axvm/src/irq/riscv.rs +++ b/virtualization/axvm/src/arch/riscv64/irq.rs @@ -27,7 +27,7 @@ use riscv_vplic::{ PLIC_CONTEXT_CLAIM_COMPLETE_OFFSET, PLIC_CONTEXT_CTRL_OFFSET, PLIC_CONTEXT_STRIDE, VPlicGlobal, }; -use super::InterruptFabric; +use crate::irq::InterruptFabric; struct RiscvPlicIrqSink { vplic: Arc, diff --git a/virtualization/axvm/src/arch/riscv64/mod.rs b/virtualization/axvm/src/arch/riscv64/mod.rs index a50f873445..76ab6c6b0a 100644 --- a/virtualization/axvm/src/arch/riscv64/mod.rs +++ b/virtualization/axvm/src/arch/riscv64/mod.rs @@ -1,7 +1,7 @@ use alloc::vec::Vec; use ax_crate_interface::impl_interface; -use ax_errno::{AxError, AxResult, ax_err}; +use ax_errno::{AxError, AxResult}; use ax_memory_addr::{PhysAddr, VirtAddr}; use axvm_types::{ AccessWidth, GuestPhysAddr, MappingFlags, NestedPagingConfig, VCpuId, VMId, VMInterruptMode, @@ -14,17 +14,25 @@ use riscv_vcpu::{ }; use riscv_vplic::host::RiscvVplicHostIf; -use super::{ - ArchOps, BoundVcpuExit, CpuUpExit, HypercallExit, MmioReadExit, MmioWriteExit, - VcpuCreateContext, VcpuRunAction, VcpuSetupContext, default_vcpu_affinities, - target_phys_cpu_ids, -}; +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; use crate::{ StopReason, + architecture::ops::default_vcpu_affinities, host::{HostMemory, default_host}, }; +mod capabilities; +#[path = "../../architecture/cpu_up.rs"] +mod cpu_up; +pub(crate) mod fdt; +mod images; +mod irq; mod npt; +mod vm; + +pub use capabilities::{host_fdt_bootarg, host_phys_to_virt}; +use cpu_up::{CpuUpExit, CpuUpOps}; +pub use images::ImageLoader; pub(crate) struct Riscv64Arch; @@ -33,10 +41,15 @@ pub(crate) enum RiscvDeferredRunWork { ExternalInterrupt { vector: usize }, } +impl CpuUpOps for Riscv64Arch { + fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { + vcpu.set_gpr(RiscvGprIndex::A0 as usize, 0); + } +} + impl ArchOps for Riscv64Arch { type VCpu = AxvmRiscvVcpu; type PerCpu = AxvmRiscvPerCpu; - type VcpuCreateState = (); type DeferredRunWork = RiscvDeferredRunWork; type NestedPageTable = npt::NestedPageTable; @@ -44,58 +57,6 @@ impl ArchOps for Riscv64Arch { riscv_vcpu::has_hardware_support() } - fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { - let mut levels = riscv_vcpu::max_guest_page_table_levels(); - for cpu_id in target_phys_cpu_ids(vcpu_mappings) { - levels = levels.min( - crate::percpu::cpu_max_guest_page_table_levels(cpu_id) - .unwrap_or_else(riscv_vcpu::max_guest_page_table_levels), - ); - } - match levels { - 3 | 4 => Ok(levels), - _ => ax_err!(Unsupported, "no supported RISC-V G-stage paging mode"), - } - } - - fn nested_paging_config( - root_paddr: PhysAddr, - levels: usize, - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - match levels { - 3 => Ok(NestedPagingConfig::new(root_paddr, 3, 41, 8)), - 4 => Ok(NestedPagingConfig::new(root_paddr, 4, 50, 9)), - _ => ax_err!(InvalidInput, "unsupported RISC-V G-stage levels"), - } - } - - fn new_nested_page_table(levels: usize) -> AxResult { - npt::NestedPageTable::new(levels) - } - - fn new_vcpu_create_state( - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - Ok(()) - } - - fn build_vcpu_create_config( - _state: &Self::VcpuCreateState, - ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig> { - Ok(RiscvVcpuCreateConfig { - hart_id: ctx.vcpu_id, - dtb_addr: ctx.dtb_addr.unwrap_or_default().as_usize(), - }) - } - - fn build_vcpu_setup_config( - _ctx: VcpuSetupContext<'_>, - ) -> AxResult<::SetupConfig> { - Ok(()) - } - fn register_platform_irq_injector() { register_platform_irq_injector(); } @@ -135,10 +96,6 @@ impl ArchOps for Riscv64Arch { vcpu.set_gpr(RiscvGprIndex::A1 as usize, arg); } - fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { - vcpu.set_gpr(RiscvGprIndex::A0 as usize, 0); - } - fn after_external_interrupt( _vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef, @@ -200,7 +157,7 @@ impl ArchOps for Riscv64Arch { target_cpu, entry_point, arg, - } => super::handle_cpu_up::( + } => cpu_up::handle::( vm, vcpu, CpuUpExit { @@ -215,19 +172,29 @@ impl ArchOps for Riscv64Arch { vm.id(), vcpu.id() ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Wait)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: true, + stop_reason: None, + })) } RiscvVmExit::Halt => { debug!("VM[{}] run VCpu[{}] Halt", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(Self::handle_halt())) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: true, + stop_reason: None, + })) } RiscvVmExit::SystemDown => { warn!("VM[{}] run VCpu[{}] SystemDown", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Stop( - StopReason::SystemDown, - ))) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: Some(StopReason::SystemDown), + })) } - RiscvVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)), + RiscvVmExit::Nothing => Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })), } } @@ -241,7 +208,10 @@ impl ArchOps for Riscv64Arch { Self::after_external_interrupt(vm, vcpu, vector); } } - Ok(VcpuRunAction::Yield) + Ok(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }) } } @@ -260,7 +230,10 @@ fn handle_riscv_nested_page_fault( vcpu.id(), ax_addr.as_usize() ); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); + return Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })); }; return Riscv64Arch::handle_vcpu_exit_bound(vm, vcpu, decoded); } @@ -276,7 +249,10 @@ fn handle_riscv_nested_page_fault( ax_addr.as_usize(), ax_flags ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } } diff --git a/virtualization/axvm/src/arch/riscv64/npt.rs b/virtualization/axvm/src/arch/riscv64/npt.rs index 631f9aab58..51b394d6f4 100644 --- a/virtualization/axvm/src/arch/riscv64/npt.rs +++ b/virtualization/axvm/src/arch/riscv64/npt.rs @@ -106,4 +106,4 @@ impl ptg::PageTableEntry for RiscvPte { } pub(crate) type NestedPageTable = - crate::arch::npt::LeveledPageTable; + crate::npt::LeveledPageTable; diff --git a/virtualization/axvm/src/arch/riscv64/vm.rs b/virtualization/axvm/src/arch/riscv64/vm.rs new file mode 100644 index 0000000000..c525ff12db --- /dev/null +++ b/virtualization/axvm/src/arch/riscv64/vm.rs @@ -0,0 +1,110 @@ +//! RISC-V VM resource creation and initialization. + +use ax_errno::{AxResult, ax_err}; +use axvm_types::{NestedPagingConfig, VmArchVcpuOps}; +use riscv_vcpu::RiscvVcpuCreateConfig; + +use super::{Riscv64Arch, irq, npt}; +use crate::{ + config::AxVMConfig, + vm::{ + AxVM, AxVMResources, + prepare::{ + PreparedVm, VmInitRequest, + address_space::{guest_owned_regions, map_guest_address_space}, + complete_vm_init, default_device_factories, + devices::PreparedDevices, + validate_guest_dtb, + vcpus::{PreparedVcpus, vcpu_placements}, + }, + }, +}; + +impl Riscv64Arch { + pub(crate) fn create_vm_resources(config: AxVMConfig) -> AxResult { + let placements = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); + let levels = guest_page_table_levels(&placements)?; + let page_table = npt::NestedPageTable::new(levels)?; + AxVMResources::from_page_table(config, page_table, |root_paddr| { + nested_paging_config(root_paddr, levels) + }) + } + + pub(crate) fn init_vm(vm: &AxVM, request: VmInitRequest<'_>) -> AxResult { + match request { + VmInitRequest::Default => { + let mut factories = default_device_factories()?; + let mode = vm.interrupt_mode(); + let emulated_devices = vm.with_config(|config| config.emu_devices().clone()); + let interrupt_fabric = irq::configure(&mut factories, mode, &emulated_devices)?; + init_vm_with(vm, &factories, interrupt_fabric) + } + VmInitRequest::Provided { + factories, + interrupt_fabric, + } => init_vm_with(vm, factories, interrupt_fabric), + } + } +} + +fn init_vm_with( + vm: &AxVM, + factories: &axdevice::DeviceFactoryRegistry, + interrupt_fabric: crate::InterruptFabric, +) -> AxResult { + complete_vm_init(vm, interrupt_fabric, |resources, interrupt_fabric| { + let placements = vcpu_placements(resources); + let dtb_addr = resources + .config() + .image_config() + .dtb_load_gpa + .unwrap_or_default(); + let vcpus = PreparedVcpus::create(vm.id(), &placements, |placement| { + Ok(RiscvVcpuCreateConfig { + hart_id: placement.id, + dtb_addr: dtb_addr.as_usize(), + }) + })?; + let mut devices = PreparedDevices::build_common(resources, factories, interrupt_fabric)?; + devices.register_special_devices(vm)?; + validate_guest_dtb(resources)?; + + let owned_regions = guest_owned_regions(resources); + map_guest_address_space(vm, resources, devices.devices(), &owned_regions)?; + vcpus.setup(resources, build_vcpu_setup_config)?; + + Ok(PreparedVm::new(vcpus, devices)) + }) +} + +fn build_vcpu_setup_config( + _config: &AxVMConfig, + _memory_regions: &[crate::vm::VMMemoryRegion], +) -> AxResult<::SetupConfig> { + Ok(()) +} + +fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> AxResult { + let mut levels = riscv_vcpu::max_guest_page_table_levels(); + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + levels = levels.min( + crate::percpu::cpu_max_guest_page_table_levels(cpu_id) + .unwrap_or_else(riscv_vcpu::max_guest_page_table_levels), + ); + } + match levels { + 3 | 4 => Ok(levels), + _ => ax_err!(Unsupported, "no supported RISC-V G-stage paging mode"), + } +} + +fn nested_paging_config( + root_paddr: ax_memory_addr::PhysAddr, + levels: usize, +) -> AxResult { + match levels { + 3 => Ok(NestedPagingConfig::new(root_paddr, 3, 41, 8)), + 4 => Ok(NestedPagingConfig::new(root_paddr, 4, 50, 9)), + _ => ax_err!(InvalidInput, "unsupported RISC-V G-stage levels"), + } +} diff --git a/virtualization/axvm/src/boot/images/x86/boot_params.rs b/virtualization/axvm/src/arch/x86_64/boot/boot_params.rs similarity index 99% rename from virtualization/axvm/src/boot/images/x86/boot_params.rs rename to virtualization/axvm/src/arch/x86_64/boot/boot_params.rs index 1906cdba4a..132c7d2b4c 100644 --- a/virtualization/axvm/src/boot/images/x86/boot_params.rs +++ b/virtualization/axvm/src/arch/x86_64/boot/boot_params.rs @@ -354,8 +354,10 @@ fn write_u64(buffer: &mut [u8], offset: usize, value: u64) { #[cfg(test)] mod tests { - use super::*; - use crate::boot::images::x86::linux::{BOOT_PARAMS_GPA, BOOT_STUB_GPA, BOOT_STUB_SIZE}; + use super::{ + super::linux::{BOOT_PARAMS_GPA, BOOT_STUB_GPA, BOOT_STUB_SIZE}, + *, + }; const SETUP_SECTS_OFFSET: usize = 0x1f1; const BOOT_FLAG_OFFSET: usize = 0x1fe; diff --git a/virtualization/axvm/src/boot/images/x86/linux.rs b/virtualization/axvm/src/arch/x86_64/boot/linux.rs similarity index 100% rename from virtualization/axvm/src/boot/images/x86/linux.rs rename to virtualization/axvm/src/arch/x86_64/boot/linux.rs diff --git a/virtualization/axvm/src/boot/images/x86/linux_boot.rs b/virtualization/axvm/src/arch/x86_64/boot/linux_boot.rs similarity index 98% rename from virtualization/axvm/src/boot/images/x86/linux_boot.rs rename to virtualization/axvm/src/arch/x86_64/boot/linux_boot.rs index 18fb82ab43..b0a8d2fac6 100644 --- a/virtualization/axvm/src/boot/images/x86/linux_boot.rs +++ b/virtualization/axvm/src/arch/x86_64/boot/linux_boot.rs @@ -103,8 +103,10 @@ fn write_u32(buffer: &mut [u8], offset: usize, value: u32) { #[cfg(test)] mod tests { - use super::*; - use crate::boot::images::x86::linux::{BOOT_PARAMS_GPA, X86LinuxHeader, X86LinuxRange}; + use super::{ + super::linux::{BOOT_PARAMS_GPA, X86LinuxHeader, X86LinuxRange}, + *, + }; const SETUP_SECTS_OFFSET: usize = 0x1f1; const BOOT_FLAG_OFFSET: usize = 0x1fe; diff --git a/virtualization/axvm/src/arch/x86_64/boot/mod.rs b/virtualization/axvm/src/arch/x86_64/boot/mod.rs new file mode 100644 index 0000000000..45d6933ac5 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/boot/mod.rs @@ -0,0 +1,547 @@ +//! x86_64 Linux, BIOS, UEFI, and MP-table image planning. + +use alloc::format; + +use ax_errno::{AxResult, ax_err_type}; +use axvm_types::GuestPhysAddr; +use axvmconfig::{EmulatedDeviceType, VMBootProtocol, VmMemMappingType}; + +use super::X86_64Arch; +use crate::{ + architecture::BootImagePlatform, + boot::{ + BootImageProvider, StaticVmImage, + images::{ImageLoaderCore, load_vm_image_from_memory}, + }, +}; + +mod boot_params; +mod linux; +mod linux_boot; +mod mptable; +mod multiboot; + +pub struct ImageLoader<'a>(ImageLoaderCore<'a>); + +impl<'a> ImageLoader<'a> { + pub fn new( + main_memory: crate::VMMemoryRegion, + config: axvmconfig::AxVMCrateConfig, + vm: crate::AxVMRef, + provider: &'a dyn BootImageProvider, + ) -> Self { + Self(ImageLoaderCore::new( + main_memory, + config, + vm, + provider, + None, + )) + } + + pub fn load(&mut self) -> AxResult { + self.0.load() + } +} + +impl BootImagePlatform for X86_64Arch { + fn default_boot_firmware_load_gpa( + config: &axvmconfig::AxVMCrateConfig, + ) -> Option { + const BUILT_IN_BIOS_LOAD_GPA: usize = 0x8000; + + (config.kernel.boot_firmware_path().is_none() + && config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot) + .then_some(GuestPhysAddr::from(BUILT_IN_BIOS_LOAD_GPA)) + } + + fn load_images_from_memory( + loader: &mut ImageLoaderCore<'_>, + images: StaticVmImage, + ) -> AxResult { + if should_direct_boot_linux(&loader.config) + && let Some(header) = detect_linux_image(images.kernel) + { + return load_linux_from_memory(loader, header, images.kernel, images.ramdisk); + } + + load_vm_image_from_memory(images.kernel, loader.kernel_load_gpa, loader.vm.clone())?; + if let Some(ramdisk) = images.ramdisk { + loader.load_ramdisk_from_memory(ramdisk)?; + } + load_boot_image_from_memory(loader, images.bios) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + fn load_images_from_filesystem(loader: &mut ImageLoaderCore<'_>) -> AxResult { + if should_direct_boot_linux(&loader.config) { + let probe = crate::boot::images::fs::kernel_read( + &loader.config, + loader.provider, + linux::HEADER_READ_SIZE, + ); + if let Ok(data) = probe + && let Some(header) = detect_linux_image(&data) + { + let kernel = crate::boot::images::fs::read_full_image( + &loader.config.kernel.kernel_path, + loader.provider, + )?; + return load_linux_from_filesystem(loader, header, &kernel); + } + } + + crate::boot::images::fs::load_vm_image( + &loader.config.kernel.kernel_path, + loader.kernel_load_gpa, + loader.vm.clone(), + loader.provider, + )?; + load_boot_image_from_filesystem(loader)?; + if let Some(ramdisk_path) = &loader.config.kernel.ramdisk_path { + loader.load_ramdisk_from_filesystem(ramdisk_path)?; + } + Ok(()) + } + + fn is_x86_linux_image_config( + config: &axvmconfig::AxVMCrateConfig, + provider: &dyn BootImageProvider, + ) -> bool { + if !should_direct_boot_linux(config) { + return false; + } + match config.kernel.image_location.as_deref() { + Some("memory") => provider + .static_vm_images() + .iter() + .find(|image| image.id == config.base.id) + .and_then(|image| detect_linux_image(image.kernel)) + .is_some(), + #[cfg(any(feature = "fs", feature = "host-fs"))] + Some("fs") => { + crate::boot::images::fs::kernel_read(config, provider, linux::HEADER_READ_SIZE) + .ok() + .and_then(|image| detect_linux_image(&image)) + .is_some() + } + _ => false, + } + } +} + +fn load_linux_from_memory( + loader: &mut ImageLoaderCore<'_>, + header: linux::X86LinuxHeader, + kernel: &[u8], + ramdisk: Option<&[u8]>, +) -> AxResult { + adjust_linux_dma_identity_layout(loader); + let payload = linux_payload(&header, kernel)?; + let initrd = ramdisk + .map(|image| { + loader + .ramdisk_load_gpa() + .map(|gpa| linux::X86LinuxRange::new(gpa.as_usize(), image.len())) + }) + .transpose()?; + let layout = linux::X86LinuxLoadLayout::new( + &header, + loader.kernel_load_gpa.as_usize(), + payload.len(), + initrd, + ) + .map_err(linux_layout_error)?; + + load_linux_layout(loader, header, layout, kernel)?; + load_vm_image_from_memory(payload, loader.kernel_load_gpa, loader.vm.clone())?; + if let Some(ramdisk) = ramdisk { + loader.load_ramdisk_from_memory(ramdisk)?; + } + Ok(()) +} + +#[cfg(any(feature = "fs", feature = "host-fs"))] +fn load_linux_from_filesystem( + loader: &mut ImageLoaderCore<'_>, + header: linux::X86LinuxHeader, + kernel: &[u8], +) -> AxResult { + adjust_linux_dma_identity_layout(loader); + let payload = linux_payload(&header, kernel)?; + let initrd = loader + .config + .kernel + .ramdisk_path + .as_deref() + .map(|path| -> AxResult<_> { + let size = crate::boot::images::fs::image_size(path, loader.provider)?; + Ok(linux::X86LinuxRange::new( + loader.ramdisk_load_gpa()?.as_usize(), + size, + )) + }) + .transpose()?; + let layout = linux::X86LinuxLoadLayout::new( + &header, + loader.kernel_load_gpa.as_usize(), + payload.len(), + initrd, + ) + .map_err(linux_layout_error)?; + + load_linux_layout(loader, header, layout, kernel)?; + load_vm_image_from_memory(payload, loader.kernel_load_gpa, loader.vm.clone())?; + if let Some(path) = &loader.config.kernel.ramdisk_path { + loader.load_ramdisk_from_filesystem(path)?; + } + Ok(()) +} + +fn load_boot_image_from_memory(loader: &ImageLoaderCore<'_>, bios: Option<&[u8]>) -> AxResult { + if !loader.config.kernel.enable_bios { + return Ok(()); + } + if let Some(bios) = bios { + let load_gpa = loader + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; + load_vm_image_from_memory(bios, load_gpa, loader.vm.clone())?; + if should_patch_multiboot_info(&loader.config) { + load_multiboot_info(loader, bios, load_gpa)?; + } + return Ok(()); + } + + if loader.config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { + return load_uefi_from_configured_path(loader); + } + if should_load_default_boot_image(loader) { + let load_gpa = builtin_bios_load_gpa(loader.bios_load_gpa)?; + load_vm_image_from_memory(multiboot::DEFAULT_BIOS_IMAGE, load_gpa, loader.vm.clone())?; + load_multiboot_info(loader, multiboot::DEFAULT_BIOS_IMAGE, load_gpa)?; + } + Ok(()) +} + +#[cfg(any(feature = "fs", feature = "host-fs"))] +fn load_boot_image_from_filesystem(loader: &ImageLoaderCore<'_>) -> AxResult { + if !loader.config.kernel.enable_bios { + return Ok(()); + } + if let Some(path) = loader.config.kernel.boot_firmware_path() { + let load_gpa = loader + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; + if should_patch_multiboot_info(&loader.config) { + let bios = crate::boot::images::fs::read_full_image(path, loader.provider)?; + validate_bios_patch_region(&bios)?; + load_vm_image_from_memory(&bios, load_gpa, loader.vm.clone())?; + load_multiboot_info(loader, &bios, load_gpa) + } else { + crate::boot::images::fs::load_vm_image( + path, + load_gpa, + loader.vm.clone(), + loader.provider, + ) + } + } else if should_load_default_boot_image(loader) { + let load_gpa = builtin_bios_load_gpa(loader.bios_load_gpa)?; + load_vm_image_from_memory(multiboot::DEFAULT_BIOS_IMAGE, load_gpa, loader.vm.clone())?; + load_multiboot_info(loader, multiboot::DEFAULT_BIOS_IMAGE, load_gpa) + } else { + Ok(()) + } +} + +fn load_uefi_from_configured_path(loader: &ImageLoaderCore<'_>) -> AxResult { + let path = loader + .config + .kernel + .boot_firmware_path() + .ok_or_else(|| ax_err_type!(NotFound, "UEFI firmware image path is missed"))?; + let load_gpa = loader + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "UEFI firmware load addr is missed"))?; + #[cfg(any(feature = "fs", feature = "host-fs"))] + { + crate::boot::images::fs::load_vm_image(path, load_gpa, loader.vm.clone(), loader.provider) + } + #[cfg(not(any(feature = "fs", feature = "host-fs")))] + { + let _ = (path, load_gpa); + ax_errno::ax_err!( + Unsupported, + "UEFI firmware path requires the fs feature when no firmware image buffer is available" + ) + } +} + +fn adjust_linux_dma_identity_layout(loader: &mut ImageLoaderCore<'_>) { + if !loader.main_memory.is_identical() { + return; + } + let memory_base = loader.main_memory.gpa.as_usize(); + loader.kernel_load_gpa = + GuestPhysAddr::from(memory_base + loader.config.kernel.kernel_load_addr); + if let Some(ramdisk_load_addr) = loader.config.kernel.ramdisk_load_addr { + loader.ramdisk_load_gpa = Some(GuestPhysAddr::from(memory_base + ramdisk_load_addr)); + } + loader.vm.with_config(|config| { + config.image_config.kernel_load_gpa = loader.kernel_load_gpa; + if let Some(load_gpa) = loader.ramdisk_load_gpa + && let Some(ramdisk) = config.image_config.ramdisk.as_mut() + { + ramdisk.load_gpa = load_gpa; + } + }); +} + +fn load_linux_layout( + loader: &ImageLoaderCore<'_>, + header: linux::X86LinuxHeader, + layout: linux::X86LinuxLoadLayout, + kernel: &[u8], +) -> AxResult { + let boot_params = build_boot_params(loader, header, layout, kernel)?; + let boot_stub = linux_boot::build_boot_image(&layout).map_err(|err| { + ax_err_type!( + InvalidInput, + format!("failed to build x86 Linux boot stub: {err:?}") + ) + })?; + load_vm_image_from_memory( + &boot_params, + layout.boot_params.start.into(), + loader.vm.clone(), + )?; + load_vm_image_from_memory(&boot_stub, layout.boot_stub.start.into(), loader.vm.clone())?; + load_vm_image_from_memory( + &mptable::build(), + mptable::MP_TABLE_GPA.into(), + loader.vm.clone(), + )?; + let entry = GuestPhysAddr::from(linux_boot::DEFAULT_LINUX_BOOT_LOAD_GPA); + loader.vm.with_config(|config| { + config.cpu_config.bsp_entry = entry; + config.cpu_config.ap_entry = entry; + }); + Ok(()) +} + +fn build_boot_params( + loader: &ImageLoaderCore<'_>, + header: linux::X86LinuxHeader, + layout: linux::X86LinuxLoadLayout, + kernel: &[u8], +) -> AxResult<[u8; linux::BOOT_PARAMS_SIZE]> { + let mut builder = boot_params::BootParamsBuilder::new( + kernel, + header, + layout, + linux::X86LinuxRange::new(loader.main_memory.gpa.as_usize(), loader.main_memory.size()), + ); + let command_line = loader.config.kernel.cmdline.as_deref().ok_or_else(|| { + ax_err_type!( + InvalidInput, + "x86 Linux direct boot requires kernel.cmdline in the VM config" + ) + })?; + builder.set_command_line(command_line).map_err(|err| { + ax_err_type!( + InvalidInput, + format!("invalid x86 Linux command line: {err:?}") + ) + })?; + for memory in &loader.config.kernel.memory_regions { + if memory.map_type == VmMemMappingType::MapAlloc { + builder.add_ram_range(linux::X86LinuxRange::new(memory.gpa, memory.size)); + } + } + for device in &loader.config.devices.passthrough_devices { + builder.add_reserved_range(linux::X86LinuxRange::new(device.base_gpa, device.length)); + } + for address in &loader.config.devices.passthrough_addresses { + builder.add_reserved_range(linux::X86LinuxRange::new(address.base_gpa, address.length)); + } + for device in &loader.config.devices.emu_devices { + if matches!(device.emu_type, EmulatedDeviceType::X86IoApic) { + builder.add_reserved_range(linux::X86LinuxRange::new(device.base_gpa, device.length)); + } + } + builder.add_reserved_range(mptable::reserved_range()); + builder.build().map_err(|err| { + ax_err_type!( + InvalidInput, + format!("failed to build x86 boot_params: {err:?}") + ) + }) +} + +fn load_multiboot_info( + loader: &ImageLoaderCore<'_>, + bios_image: &[u8], + bios_load_gpa: GuestPhysAddr, +) -> AxResult { + const INFO_GPA: usize = 0x6000; + const MMAP_GPA: usize = 0x6040; + let mem_base = loader.main_memory.gpa.as_usize() as u64; + let mem_size = loader.main_memory.size() as u64; + let mut info = [0u8; 52]; + write_u32(&mut info, 0, (1 << 0) | (1 << 6)); + write_u32(&mut info, 4, 639); + write_u32( + &mut info, + 8, + (mem_size.saturating_sub(0x100000) / 1024) as u32, + ); + write_u32(&mut info, 44, 24); + write_u32(&mut info, 48, MMAP_GPA as u32); + let mut mmap = [0u8; 24]; + write_u32(&mut mmap, 0, 20); + write_u64(&mut mmap, 4, mem_base); + write_u64(&mut mmap, 12, mem_size); + write_u32(&mut mmap, 20, 1); + validate_bios_patch_region(bios_image)?; + load_vm_image_from_memory(&info, INFO_GPA.into(), loader.vm.clone())?; + load_vm_image_from_memory(&mmap, MMAP_GPA.into(), loader.vm.clone())?; + load_vm_image_from_memory( + &(INFO_GPA as u32).to_le_bytes(), + (bios_load_gpa.as_usize() + multiboot::AXVM_BIOS_EBX_IMM_OFFSET).into(), + loader.vm.clone(), + ) +} + +fn should_direct_boot_linux(config: &axvmconfig::AxVMCrateConfig) -> bool { + !config.kernel.enable_bios && config.kernel.effective_boot_protocol() == VMBootProtocol::Direct +} + +fn should_patch_multiboot_info(config: &axvmconfig::AxVMCrateConfig) -> bool { + config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot +} + +fn should_load_default_boot_image(loader: &ImageLoaderCore<'_>) -> bool { + loader.config.kernel.enable_bios + && loader.config.kernel.boot_firmware_path().is_none() + && loader.config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot +} + +fn detect_linux_image(image: &[u8]) -> Option { + linux::X86LinuxHeader::parse(image).ok() +} + +fn linux_payload<'a>(header: &linux::X86LinuxHeader, image: &'a [u8]) -> AxResult<&'a [u8]> { + image.get(header.payload_offset()..).ok_or_else(|| { + ax_err_type!( + InvalidInput, + format!( + "x86 Linux bzImage payload offset {:#x} exceeds image size {:#x}", + header.payload_offset(), + image.len() + ) + ) + }) +} + +fn linux_layout_error(err: linux::X86LinuxLayoutError) -> ax_errno::AxError { + ax_err_type!( + InvalidInput, + format!("invalid x86 Linux memory layout: {err:?}") + ) +} + +fn builtin_bios_load_gpa(configured: Option) -> AxResult { + let default = GuestPhysAddr::from(multiboot::DEFAULT_BIOS_LOAD_GPA); + match configured { + Some(gpa) if gpa != default => Err(ax_err_type!( + InvalidInput, + format!( + "built-in x86 BIOS must be loaded at GPA {:#x}, but bios_load_addr is {:#x}", + default.as_usize(), + gpa.as_usize() + ) + )), + Some(gpa) => Ok(gpa), + None => Ok(default), + } +} + +fn validate_bios_patch_region(bios: &[u8]) -> AxResult { + let patch_end = multiboot::AXVM_BIOS_EBX_IMM_OFFSET + core::mem::size_of::(); + if bios.len() < patch_end + || bios[multiboot::AXVM_BIOS_EBX_IMM_OFFSET - 1] != multiboot::MOV_EBX_IMM32_OPCODE + { + return Err(ax_err_type!( + InvalidInput, + "x86 BIOS image does not match the AxVM multiboot patch layout" + )); + } + Ok(()) +} + +fn write_u32(buffer: &mut [u8], offset: usize, value: u32) { + buffer[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn write_u64(buffer: &mut [u8], offset: usize, value: u64) { + buffer[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn built_in_bios_uses_default_gpa_when_unspecified() { + assert_eq!( + builtin_bios_load_gpa(None).unwrap(), + GuestPhysAddr::from(multiboot::DEFAULT_BIOS_LOAD_GPA) + ); + } + + #[test] + fn built_in_bios_accepts_explicit_default_gpa() { + let default = GuestPhysAddr::from(multiboot::DEFAULT_BIOS_LOAD_GPA); + assert_eq!(builtin_bios_load_gpa(Some(default)).unwrap(), default); + } + + #[test] + fn built_in_bios_rejects_non_default_gpa() { + let invalid = GuestPhysAddr::from(multiboot::DEFAULT_BIOS_LOAD_GPA + 0x1000); + assert!(builtin_bios_load_gpa(Some(invalid)).is_err()); + } + + #[test] + fn legacy_bios_config_uses_multiboot_patch() { + let mut config = axvmconfig::AxVMCrateConfig::default(); + config.kernel.enable_bios = true; + assert!(should_patch_multiboot_info(&config)); + } + + #[test] + fn uefi_config_skips_multiboot_patch() { + let mut config = axvmconfig::AxVMCrateConfig::default(); + config.kernel.enable_bios = true; + config.kernel.boot_protocol = Some(VMBootProtocol::Uefi); + assert!(!should_patch_multiboot_info(&config)); + } + + #[test] + fn linux_direct_boot_requires_direct_protocol_without_bios() { + let mut config = axvmconfig::AxVMCrateConfig::default(); + assert!(should_direct_boot_linux(&config)); + + config.kernel.enable_bios = true; + assert!(!should_direct_boot_linux(&config)); + + config.kernel.boot_protocol = Some(VMBootProtocol::Uefi); + assert!(!should_direct_boot_linux(&config)); + + config.kernel.boot_protocol = Some(VMBootProtocol::Direct); + assert!(!should_direct_boot_linux(&config)); + + config.kernel.enable_bios = false; + assert!(should_direct_boot_linux(&config)); + } +} diff --git a/virtualization/axvm/src/boot/images/x86/mptable.rs b/virtualization/axvm/src/arch/x86_64/boot/mptable.rs similarity index 92% rename from virtualization/axvm/src/boot/images/x86/mptable.rs rename to virtualization/axvm/src/arch/x86_64/boot/mptable.rs index 1c228d22e4..5a134dd123 100644 --- a/virtualization/axvm/src/boot/images/x86/mptable.rs +++ b/virtualization/axvm/src/arch/x86_64/boot/mptable.rs @@ -22,10 +22,6 @@ const IO_APIC_VERSION: u8 = 0x11; const BUS_ID_PCI: u8 = 0; const BUS_ID_ISA: u8 = 1; -pub const QEMU_PASSTHROUGH_BLOCK_DEVICE: u8 = 3; -pub const QEMU_PASSTHROUGH_BLOCK_FUNCTION: u8 = 0; -pub const QEMU_PASSTHROUGH_BLOCK_PIN: u8 = 1; - const MP_IRQ_FLAGS_CONFORMING: u16 = 0; const MP_IRQ_FLAGS_ACTIVE_LOW: u16 = 0x3; const MP_IRQ_FLAGS_LEVEL_TRIGGERED: u16 = 0xc; @@ -169,13 +165,6 @@ const fn pci_intx_gsi(dev: u8, pin: u8) -> u8 { 16 + ((dev + pin) & 3) } -pub const fn qemu_passthrough_block_gsi() -> usize { - pci_intx_gsi( - QEMU_PASSTHROUGH_BLOCK_DEVICE, - QEMU_PASSTHROUGH_BLOCK_PIN - 1, - ) as usize -} - fn interrupt_entry( interrupt_type: u8, flags: u16, @@ -225,19 +214,20 @@ mod tests { #[test] fn pci_intx_entries_are_low_active_level_triggered() { - let source_irq = (QEMU_PASSTHROUGH_BLOCK_DEVICE << 2) | (QEMU_PASSTHROUGH_BLOCK_PIN - 1); + let (device, _, pin, guest_gsi) = crate::boot::x86_qemu_passthrough_block_intx(); + let source_irq = (device << 2) | (pin - 1); let entry = interrupt_entry( 0, PCI_INTX_IRQ_FLAGS, BUS_ID_PCI, source_irq, - qemu_passthrough_block_gsi() as u8, + guest_gsi as u8, ); assert_eq!(u16::from_le_bytes([entry[2], entry[3]]), 0x0f); assert_eq!(entry[4], BUS_ID_PCI); assert_eq!(entry[5], source_irq); - assert_eq!(entry[7], qemu_passthrough_block_gsi() as u8); + assert_eq!(entry[7], guest_gsi as u8); } #[test] diff --git a/virtualization/axvm/src/boot/images/x86/multiboot.rs b/virtualization/axvm/src/arch/x86_64/boot/multiboot.rs similarity index 100% rename from virtualization/axvm/src/boot/images/x86/multiboot.rs rename to virtualization/axvm/src/arch/x86_64/boot/multiboot.rs diff --git a/virtualization/axvm/src/arch/x86_64/capabilities.rs b/virtualization/axvm/src/arch/x86_64/capabilities.rs new file mode 100644 index 0000000000..685d12a588 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/capabilities.rs @@ -0,0 +1,8 @@ +//! x86_64 implementations of AxVM platform capability hooks. + +use super::X86_64Arch; +use crate::architecture::{GuestBootPlatform, HostTimePlatform}; + +impl HostTimePlatform for X86_64Arch {} + +impl GuestBootPlatform for X86_64Arch {} diff --git a/virtualization/axvm/src/arch/x86_64/exit.rs b/virtualization/axvm/src/arch/x86_64/exit.rs new file mode 100644 index 0000000000..4f50ef0437 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/exit.rs @@ -0,0 +1,78 @@ +//! x86-only port, nested-fault, and deferred exit handling. + +use ax_errno::AxResult; +use axvm_types::{AccessWidth, GuestPhysAddr, MappingFlags, Port}; + +use super::{ArchOps, AxvmX86Vcpu, X86_64Arch}; +use crate::architecture::{BoundVcpuExit, VcpuRunAction}; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum DeferredRunWork { + ExternalInterrupt { vector: usize }, + PreemptionTimer, + InterruptEnd { vector: Option }, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct IoReadExit { + pub(crate) port: Port, + pub(crate) width: AccessWidth, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct IoWriteExit { + pub(crate) port: Port, + pub(crate) width: AccessWidth, + pub(crate) data: u64, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct NestedPageFaultExit { + pub(crate) addr: GuestPhysAddr, + pub(crate) access_flags: MappingFlags, +} + +pub(crate) fn handle_io_read( + vm: &crate::AxVM, + vcpu: &crate::vm::AxVCpuRef, + exit: IoReadExit, +) -> AxResult> { + let val = vm.get_devices()?.handle_port_read(exit.port, exit.width)?; + vcpu.set_gpr(0, val); + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn handle_io_write( + vm: &crate::AxVM, + exit: IoWriteExit, +) -> AxResult> { + vm.get_devices()? + .handle_port_write(exit.port, exit.width, exit.data as usize)?; + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn finish( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + work: DeferredRunWork, +) -> AxResult { + match work { + DeferredRunWork::ExternalInterrupt { vector } => { + X86_64Arch::after_external_interrupt(vm, vcpu, vector); + } + DeferredRunWork::PreemptionTimer => { + crate::timer::check_events(); + super::irq::inject_due_pit_irq0(vm, vcpu); + super::irq::inject_pending_serial_irq(vm, vcpu); + } + DeferredRunWork::InterruptEnd { vector } => { + if let Some(vector) = vector { + super::irq::inject_pending_ioapic_irq_after_eoi(vm, vcpu, vector); + } + } + } + Ok(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + }) +} diff --git a/virtualization/axvm/src/arch/x86_64/fdt.rs b/virtualization/axvm/src/arch/x86_64/fdt.rs new file mode 100644 index 0000000000..7011d0f0c9 --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/fdt.rs @@ -0,0 +1,4 @@ +//! x86_64 compatibility marker for the target-selected FDT facade. + +#[doc(hidden)] +pub const SUPPORTED: bool = false; diff --git a/virtualization/axvm/src/host/irq.rs b/virtualization/axvm/src/arch/x86_64/host_irq.rs similarity index 71% rename from virtualization/axvm/src/host/irq.rs rename to virtualization/axvm/src/arch/x86_64/host_irq.rs index 15d65dd6c5..0da6411cd3 100644 --- a/virtualization/axvm/src/host/irq.rs +++ b/virtualization/axvm/src/arch/x86_64/host_irq.rs @@ -3,23 +3,23 @@ #[cfg(test)] use core::sync::atomic::{AtomicUsize, Ordering}; -use super::arceos; +use ax_std::os::arceos::modules::ax_hal::irq as host_irq; -pub(crate) type IrqContext = arceos::ArceOsIrqContext; -pub(crate) type IrqError = arceos::ArceOsIrqError; -pub(crate) type IrqId = arceos::ArceOsIrqId; -pub(crate) type IrqReturn = arceos::ArceOsIrqReturn; -pub(crate) type IrqSource = arceos::ArceOsIrqSource; +pub(crate) type IrqContext = host_irq::IrqContext; +pub(crate) type IrqError = host_irq::IrqError; +pub(crate) type IrqId = host_irq::IrqId; +pub(crate) type IrqReturn = host_irq::IrqReturn; +pub(crate) type IrqSource = host_irq::IrqSource; pub(crate) fn make_irq_id(domain: u16, hwirq: u32) -> IrqId { - arceos::make_irq_id(domain, hwirq) + host_irq::IrqId::new(host_irq::IrqDomainId(domain), host_irq::HwIrq(hwirq)) } pub(crate) fn request_shared_irq( irq: IrqId, handler: impl FnMut(IrqContext) -> IrqReturn + Send + 'static, -) -> Result { - arceos::request_shared_irq(irq, handler) +) -> Result { + host_irq::request_shared_irq(irq, handler) } #[cfg(test)] @@ -41,11 +41,11 @@ fn set_host_irq_enable_impl(irq: IrqId, enabled: bool) -> Result<(), IrqError> { #[cfg(not(test))] fn set_host_irq_enable_impl(irq: IrqId, enabled: bool) -> Result<(), IrqError> { - arceos::set_irq_enable(irq, enabled) + host_irq::set_enable(irq, enabled) } -pub(crate) fn resolve_irq_source(source: IrqSource) -> Result { - arceos::resolve_irq_source(source) +pub(crate) fn resolve_irq_source(source: IrqSource) -> Result { + host_irq::resolve_irq_source(source) } #[cfg(test)] diff --git a/virtualization/axvm/src/runtime/x86_irq.rs b/virtualization/axvm/src/arch/x86_64/irq.rs similarity index 92% rename from virtualization/axvm/src/runtime/x86_irq.rs rename to virtualization/axvm/src/arch/x86_64/irq.rs index 1f5013ee2e..970b5862bc 100644 --- a/virtualization/axvm/src/runtime/x86_irq.rs +++ b/virtualization/axvm/src/arch/x86_64/irq.rs @@ -1,11 +1,12 @@ use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use ax_kspin::SpinRaw as Mutex; +use axvm_types::VmArchVcpuOps; use crate::{ InterruptTriggerMode, + arch::x86_64::host_irq::{self as irq, IrqSource}, config::VMInterruptMode, - host::irq::{self, IrqSource}, runtime::{VCpuRef, VMRef}, }; @@ -93,7 +94,7 @@ pub fn inject_due_pit_irq0(vm: &VMRef, vcpu: &VCpuRef) { return; } - let now_ns = crate::host::arceos::monotonic_time_nanos(); + let now_ns = ax_std::os::arceos::modules::ax_hal::time::monotonic_time_nanos(); let Ok(devices) = vm.get_devices() else { return; }; @@ -106,15 +107,16 @@ pub fn inject_due_pit_irq0(vm: &VMRef, vcpu: &VCpuRef) { return; }; - vcpu.inject_interrupt_with_trigger( - irq.vector as _, - if irq.level_triggered { - InterruptTriggerMode::LevelTriggered - } else { - InterruptTriggerMode::EdgeTriggered - }, - ) - .unwrap(); + vcpu.get_arch_vcpu() + .inject_interrupt_with_trigger( + irq.vector as _, + if irq.level_triggered { + InterruptTriggerMode::LevelTriggered + } else { + InterruptTriggerMode::EdgeTriggered + }, + ) + .unwrap(); } pub fn inject_pending_serial_irq(vm: &VMRef, vcpu: &VCpuRef) { @@ -135,15 +137,16 @@ pub fn inject_pending_serial_irq(vm: &VMRef, vcpu: &VCpuRef) { }; trace!("Injecting x86 COM1 RX IRQ vector {:#x}", irq.vector); - vcpu.inject_interrupt_with_trigger( - irq.vector as _, - if irq.level_triggered { - InterruptTriggerMode::LevelTriggered - } else { - InterruptTriggerMode::EdgeTriggered - }, - ) - .unwrap(); + vcpu.get_arch_vcpu() + .inject_interrupt_with_trigger( + irq.vector as _, + if irq.level_triggered { + InterruptTriggerMode::LevelTriggered + } else { + InterruptTriggerMode::EdgeTriggered + }, + ) + .unwrap(); } pub fn inject_pending_ioapic_irq_after_eoi(vm: &VMRef, vcpu: &VCpuRef, vector: u8) { @@ -170,15 +173,16 @@ pub fn inject_pending_ioapic_irq_after_eoi(vm: &VMRef, vcpu: &VCpuRef, vector: u "Injecting pending x86 IOAPIC level IRQ vector {:#x} after EOI {vector:#x}", irq.vector ); - vcpu.inject_interrupt_with_trigger( - irq.vector as _, - if irq.level_triggered { - InterruptTriggerMode::LevelTriggered - } else { - InterruptTriggerMode::EdgeTriggered - }, - ) - .unwrap(); + vcpu.get_arch_vcpu() + .inject_interrupt_with_trigger( + irq.vector as _, + if irq.level_triggered { + InterruptTriggerMode::LevelTriggered + } else { + InterruptTriggerMode::EdgeTriggered + }, + ) + .unwrap(); } fn should_rearm_forwarded_host_gsi_after_eoi(pending: Option) -> bool { @@ -436,15 +440,16 @@ fn forward_passthrough_gsi( return false; }; - vcpu.inject_interrupt_with_trigger( - guest_irq.vector as _, - if guest_irq.level_triggered { - InterruptTriggerMode::LevelTriggered - } else { - InterruptTriggerMode::EdgeTriggered - }, - ) - .unwrap(); + vcpu.get_arch_vcpu() + .inject_interrupt_with_trigger( + guest_irq.vector as _, + if guest_irq.level_triggered { + InterruptTriggerMode::LevelTriggered + } else { + InterruptTriggerMode::EdgeTriggered + }, + ) + .unwrap(); true } @@ -621,7 +626,7 @@ mod tests { IOAPIC_IRQ_PENDING_LEVEL.store(0, Ordering::Release); IOAPIC_IRQ_MASKED.store(0, Ordering::Release); IOAPIC_IRQ_ACTIVATED.store(0, Ordering::Release); - crate::host::irq::reset_test_irq_enable_state(); + crate::arch::x86_64::host_irq::reset_test_irq_enable_state(); for activator in IOAPIC_IRQ_ACTIVATORS.iter() { *activator.lock() = None; } @@ -664,7 +669,7 @@ mod tests { #[test] fn host_irq_storage_preserves_domain_and_hwirq() { - let irq = crate::host::irq::make_irq_id(2, 18); + let irq = crate::arch::x86_64::host_irq::make_irq_id(2, 18); assert_eq!(raw_to_host_irq(host_irq_to_raw(irq)), irq); } @@ -673,7 +678,7 @@ mod tests { with_clean_forwarding_routes(|| { let fallback_guest_gsi = 7; let explicit_guest_gsi = 18; - let host_irq = crate::host::irq::make_irq_id(2, 7); + let host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, 7); IOAPIC_HOST_IRQS[fallback_guest_gsi] .store(host_irq_to_raw(host_irq), Ordering::Release); @@ -688,7 +693,7 @@ mod tests { with_clean_forwarding_routes(|| { let fallback_guest_gsi = 10; let explicit_guest_gsi = 18; - let host_irq = crate::host::irq::make_irq_id(2, 10); + let host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, 10); IOAPIC_HOST_IRQS[fallback_guest_gsi] .store(host_irq_to_raw(host_irq), Ordering::Release); @@ -710,8 +715,8 @@ mod tests { with_clean_forwarding_routes(|| { let low_level_gsi = COM1_GSI; let high_edge_gsi = 18; - let low_host_irq = crate::host::irq::make_irq_id(2, low_level_gsi as u32); - let high_host_irq = crate::host::irq::make_irq_id(2, high_edge_gsi as u32); + let low_host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, low_level_gsi as u32); + let high_host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, high_edge_gsi as u32); register_ioapic_irq_forwarding_route_with_trigger( low_level_gsi, @@ -755,7 +760,7 @@ mod tests { fn forwarding_activator_drops_pre_activation_pending_state() { with_clean_forwarding_routes(|| { let guest_gsi = 18; - let host_irq = crate::host::irq::make_irq_id(2, 10); + let host_irq = crate::arch::x86_64::host_irq::make_irq_id(2, 10); ACTIVATION_COUNT.store(0, Ordering::Release); register_ioapic_irq_forwarding_route(guest_gsi, host_irq); register_ioapic_irq_forwarding_activator(guest_gsi, count_activation); @@ -768,7 +773,7 @@ mod tests { forwarded_ioapic_gsi_state_for_test(guest_gsi), (false, false, false) ); - assert!(crate::host::irq::test_irq_is_enabled(host_irq)); + assert!(crate::arch::x86_64::host_irq::test_irq_is_enabled(host_irq)); }); } diff --git a/virtualization/axvm/src/arch/x86_64/mod.rs b/virtualization/axvm/src/arch/x86_64/mod.rs index 5318398bf0..aa5bbe4bf5 100644 --- a/virtualization/axvm/src/arch/x86_64/mod.rs +++ b/virtualization/axvm/src/arch/x86_64/mod.rs @@ -22,19 +22,28 @@ use x86_vlapic::{ X86VlapicResult, X86VmId, }; -use super::{ - ArchOps, BoundVcpuExit, HypercallExit, IoReadExit, IoWriteExit, LegacyDeferredRunWork, - MmioReadExit, MmioWriteExit, NestedPageFaultExit, SysRegReadExit, SysRegWriteExit, - VcpuCreateContext, VcpuRunAction, VcpuSetupContext, -}; +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; use crate::{ StopReason, - host::{HostConsole, HostMemory, HostTime, default_host}, + host::{HostMemory, default_host}, manager, vcpu::get_current_vcpu, }; +pub(crate) mod boot; +mod capabilities; +mod exit; +pub(crate) mod fdt; +mod host_irq; +pub(crate) mod irq; mod npt; +pub(crate) mod port; +#[path = "../../architecture/sysreg.rs"] +mod sysreg; +mod vm; + +use exit::{DeferredRunWork, IoReadExit, IoWriteExit, NestedPageFaultExit}; +use sysreg::{SysRegReadExit, SysRegWriteExit}; const QEMU_EXIT_PORT: u16 = 0x604; const QEMU_EXIT_MAGIC: u64 = 0x2000; @@ -42,65 +51,23 @@ const RFLAGS_INTERRUPT_FLAG: u64 = 1 << 9; pub(crate) struct X86_64Arch; -pub(crate) struct X86VcpuCreateState; - impl ArchOps for X86_64Arch { type VCpu = AxvmX86Vcpu; type PerCpu = AxvmX86PerCpu; - type VcpuCreateState = X86VcpuCreateState; - type DeferredRunWork = LegacyDeferredRunWork; + type DeferredRunWork = DeferredRunWork; type NestedPageTable = npt::NestedPageTable; fn has_hardware_support() -> bool { x86_vcpu::has_hardware_support() } - fn new_vcpu_create_state( - _vcpu_mappings: &[(usize, Option, usize)], - ) -> AxResult { - Ok(X86VcpuCreateState) - } - - fn build_vcpu_create_config( - _state: &Self::VcpuCreateState, - _ctx: VcpuCreateContext, - ) -> AxResult<::CreateConfig> { - Ok(X86VCpuCreateConfig) - } - - fn build_vcpu_setup_config( - ctx: VcpuSetupContext<'_>, - ) -> AxResult<::SetupConfig> { - let mut config = X86VCpuSetupConfig { - emulate_com1: ctx.emulates_console, - guest_memory_regions: ctx - .memory_regions - .iter() - .map(|region| x86_vcpu::X86GuestMemoryRegion { - gpa: X86GuestPhysAddr::from_usize(region.gpa.as_usize()), - hva: X86HostVirtAddr::from_usize(region.hva.as_usize()), - size: region.size(), - }) - .collect(), - ..Default::default() - }; - for port in ctx.passthrough_ports { - x86_result(config.add_passthrough_port_range(port.base, port.length))?; - } - Ok(config) - } - - fn new_nested_page_table(levels: usize) -> AxResult { - npt::NestedPageTable::new(levels) - } - fn before_first_run(vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { - crate::runtime::x86_irq::enable_ioapic_irq_forwarding(vm, vcpu); + irq::enable_ioapic_irq_forwarding(vm, vcpu); } fn before_vcpu_run(vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { - crate::runtime::x86_irq::drain_pending_ioapic_irqs(vm, vcpu); - crate::runtime::x86_irq::activate_ready_ioapic_forwarding_routes(vm); + irq::drain_pending_ioapic_irqs(vm, vcpu); + irq::activate_ready_ioapic_forwarding_routes(vm); } fn after_external_interrupt( @@ -110,31 +77,11 @@ impl ArchOps for X86_64Arch { ) { crate::host::arceos::dispatch_host_irq(vector); crate::check_timer_events(); - crate::runtime::x86_irq::inject_pending_serial_irq(vm, vcpu); - } - - fn after_preemption_timer(vm: &crate::AxVMRef, vcpu: &crate::vm::AxVCpuRef) { - crate::timer::check_events(); - crate::runtime::x86_irq::inject_due_pit_irq0(vm, vcpu); - crate::runtime::x86_irq::inject_pending_serial_irq(vm, vcpu); - } - - fn after_interrupt_end( - vm: &crate::AxVMRef, - vcpu: &crate::vm::AxVCpuRef, - vector: Option, - ) { - if let Some(vector) = vector { - crate::runtime::x86_irq::inject_pending_ioapic_irq_after_eoi(vm, vcpu, vector); - } - } - - fn handle_halt() -> VcpuRunAction { - VcpuRunAction::Yield + irq::inject_pending_serial_irq(vm, vcpu); } fn on_last_vcpu_exit(vm_id: usize) { - crate::runtime::x86_irq::disable_ioapic_irq_forwarding_for_vm(vm_id); + irq::disable_ioapic_irq_forwarding_for_vm(vm_id); } fn handle_vcpu_exit_bound( @@ -146,7 +93,7 @@ impl ArchOps for X86_64Arch { X86VmExit::Hypercall { nr, args } => { super::handle_hypercall(vm, vcpu, HypercallExit { nr, args }) } - X86VmExit::PortIoRead { port, width } => super::handle_io_read::( + X86VmExit::PortIoRead { port, width } => exit::handle_io_read( vm, vcpu, IoReadExit { @@ -157,11 +104,12 @@ impl ArchOps for X86_64Arch { X86VmExit::PortIoWrite { port, width, data } => { if x86_qemu_shutdown_port(port, width, data) { warn!("VM[{}] run VCpu[{}] SystemDown", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Stop( - StopReason::SystemDown, - ))) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: Some(StopReason::SystemDown), + })) } else { - super::handle_io_write( + exit::handle_io_write( vm, IoWriteExit { port: x86_port_to_ax(port), @@ -196,7 +144,7 @@ impl ArchOps for X86_64Arch { data, }, ), - X86VmExit::MsrRead { addr } => super::handle_sys_reg_read( + X86VmExit::MsrRead { addr } => sysreg::handle_read( vm, vcpu, SysRegReadExit { @@ -204,47 +152,47 @@ impl ArchOps for X86_64Arch { reg: 0, }, ), - X86VmExit::MsrWrite { addr, value } => super::handle_sys_reg_write( + X86VmExit::MsrWrite { addr, value } => sysreg::handle_write( vm, SysRegWriteExit { addr: x86_msr_addr_to_ax(addr), value, }, ), - X86VmExit::NestedPageFault { addr, access_flags } => { - handle_x86_nested_page_fault::( - vm, - NestedPageFaultExit { - addr: x86_guest_phys_addr_to_ax(addr), - access_flags: x86_access_flags_to_ax(access_flags), - }, - ) - } + X86VmExit::NestedPageFault { addr, access_flags } => handle_x86_nested_page_fault( + vm, + NestedPageFaultExit { + addr: x86_guest_phys_addr_to_ax(addr), + access_flags: x86_access_flags_to_ax(access_flags), + }, + ), X86VmExit::ExternalInterrupt { vector } => { debug!("VM[{}] run VCpu[{}] get irq {vector}", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Defer( - LegacyDeferredRunWork::ExternalInterrupt { - vector: vector as usize, - }, - )) + Ok(BoundVcpuExit::Defer(DeferredRunWork::ExternalInterrupt { + vector: vector as usize, + })) } X86VmExit::PreemptionTimer => { - Ok(BoundVcpuExit::Defer(LegacyDeferredRunWork::PreemptionTimer)) + Ok(BoundVcpuExit::Defer(DeferredRunWork::PreemptionTimer)) } X86VmExit::InterruptEnd { vector } => { - Ok(BoundVcpuExit::Defer(LegacyDeferredRunWork::InterruptEnd { + Ok(BoundVcpuExit::Defer(DeferredRunWork::InterruptEnd { vector, })) } X86VmExit::Halt => { debug!("VM[{}] run VCpu[{}] Halt", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(Self::handle_halt())) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } X86VmExit::SystemDown => { warn!("VM[{}] run VCpu[{}] SystemDown", vm.id(), vcpu.id()); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Stop( - StopReason::SystemDown, - ))) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: Some(StopReason::SystemDown), + })) } X86VmExit::FailEntry { hardware_entry_failure_reason, @@ -254,7 +202,10 @@ impl ArchOps for X86_64Arch { vm.id(), vcpu.id() ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } X86VmExit::Nothing => Ok(BoundVcpuExit::Continue), _ => Err(AxError::Unsupported), @@ -266,7 +217,7 @@ impl ArchOps for X86_64Arch { vcpu: &crate::vm::AxVCpuRef, work: Self::DeferredRunWork, ) -> AxResult { - super::finish_legacy_deferred_run_work::(vm, vcpu, work) + exit::finish(vm, vcpu, work) } } @@ -294,26 +245,26 @@ impl X86VlapicHostOps for AxvmX86HostOps { } fn current_time_nanos() -> u64 { - crate::host::arceos::monotonic_time_nanos() + ax_std::os::arceos::modules::ax_hal::time::monotonic_time_nanos() } fn register_timer(deadline_nanos: u64, callback: X86TimerCallback) -> Option { - Some(default_host().register_timer( + Some(crate::timer::register_timer( deadline_nanos, Box::new(move |deadline: Duration| callback(deadline.as_nanos() as u64)), )) } fn cancel_timer(token: usize) { - default_host().cancel_timer(token); + crate::timer::cancel_timer(token); } fn write_bytes(bytes: &[u8]) { - default_host().write_bytes(bytes); + ax_std::os::arceos::modules::ax_hal::console::write_bytes(bytes); } fn read_bytes(bytes: &mut [u8]) -> usize { - default_host().read_bytes(bytes) + ax_std::os::arceos::modules::ax_hal::console::read_bytes(bytes) } fn current_vm_id() -> X86VmId { @@ -385,7 +336,7 @@ impl X86HostOps for AxvmX86HostOps { } fn nanos_to_ticks(nanos: u64) -> u64 { - default_host().nanos_to_ticks(nanos) + ax_std::os::arceos::modules::ax_hal::time::nanos_to_ticks(nanos) } fn poll_host_interrupt() -> Option { @@ -526,20 +477,20 @@ pub(crate) fn x86_apic_access_page_addr() -> axvm_types::HostPhysAddr { axvm_types::HostPhysAddr::from(addr.as_usize()) } -fn handle_x86_nested_page_fault( +fn handle_x86_nested_page_fault( vm: &crate::AxVMRef, exit: NestedPageFaultExit, -) -> AxResult> -where - A: ArchOps, -{ +) -> AxResult> { if vm.get_devices()?.find_mmio_dev(exit.addr).is_some() { warn!( "VM[{}] nested page fault at {:#x} maps MMIO but x86 core did not decode it", vm.id(), exit.addr.as_usize() ); - return Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)); + return Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })); } if vm.handle_nested_page_fault(exit.addr, exit.access_flags) { @@ -551,7 +502,10 @@ where exit.addr.as_usize(), exit.access_flags ); - Ok(BoundVcpuExit::Complete(VcpuRunAction::Yield)) + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) } } diff --git a/virtualization/axvm/src/arch/x86_64/npt.rs b/virtualization/axvm/src/arch/x86_64/npt.rs index 906bbecfbf..e097289f86 100644 --- a/virtualization/axvm/src/arch/x86_64/npt.rs +++ b/virtualization/axvm/src/arch/x86_64/npt.rs @@ -327,12 +327,8 @@ pub type ExtendedPageTableEntry = EPTEntry; #[cfg(feature = "svm")] pub type ExtendedPageTableEntry = NPTEntry; -pub(crate) type NestedPageTable = crate::arch::npt::LeveledPageTable< - ExtendedPageTableMetadata, - ExtendedPageTableMetadata, - H, - false, ->; +pub(crate) type NestedPageTable = + crate::npt::LeveledPageTable; fn config_to_flags(config: ptg::PteConfig) -> MappingFlags { let mut flags = MappingFlags::empty(); diff --git a/virtualization/axvm/src/host/x86_port.rs b/virtualization/axvm/src/arch/x86_64/port.rs similarity index 100% rename from virtualization/axvm/src/host/x86_port.rs rename to virtualization/axvm/src/arch/x86_64/port.rs diff --git a/virtualization/axvm/src/arch/x86_64/vm.rs b/virtualization/axvm/src/arch/x86_64/vm.rs new file mode 100644 index 0000000000..f852a8aa2b --- /dev/null +++ b/virtualization/axvm/src/arch/x86_64/vm.rs @@ -0,0 +1,173 @@ +//! x86_64 VM resource creation and initialization. + +use alloc::sync::Arc; + +use ax_errno::{AxResult, ax_err_type}; +#[cfg(feature = "vmx")] +use ax_memory_addr::PAGE_SIZE_4K; +use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; +#[cfg(feature = "vmx")] +use axvm_types::MappingFlags; +use axvm_types::{EmulatedDeviceType, NestedPagingConfig, VmArchVcpuOps}; +use x86_vcpu::{ + X86GuestMemoryRegion, X86GuestPhysAddr, X86HostVirtAddr, X86VCpuCreateConfig, + X86VCpuSetupConfig, +}; + +#[cfg(feature = "vmx")] +use super::x86_apic_access_page_addr; +use super::{X86_64Arch, npt, x86_result}; +use crate::{ + config::AxVMConfig, + layout::GuestOwnedRegion, + vm::{ + AxVM, AxVMResources, + prepare::{ + PreparedVm, VmInitRequest, + address_space::{guest_owned_regions, map_guest_address_space}, + complete_vm_init, default_device_factories, + devices::PreparedDevices, + validate_guest_dtb, + vcpus::{PreparedVcpus, vcpu_placements}, + }, + }, +}; + +impl X86_64Arch { + pub(crate) fn create_vm_resources(config: AxVMConfig) -> AxResult { + let placements = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); + let levels = guest_page_table_levels(&placements); + let page_table = npt::NestedPageTable::new(levels)?; + AxVMResources::from_page_table(config, page_table, |root_paddr| { + let gpa_bits = match levels { + 3 => 39, + 4 => 48, + _ => { + return ax_errno::ax_err!( + InvalidInput, + "unsupported x86 nested page-table levels" + ); + } + }; + Ok(NestedPagingConfig::new(root_paddr, levels, gpa_bits, 0)) + }) + } + + pub(crate) fn init_vm(vm: &AxVM, request: VmInitRequest<'_>) -> AxResult { + match request { + VmInitRequest::Default => { + let factories = default_device_factories()?; + let interrupt_fabric = crate::InterruptFabric::new(vm.interrupt_mode()); + init_vm_with(vm, &factories, interrupt_fabric) + } + VmInitRequest::Provided { + factories, + interrupt_fabric, + } => init_vm_with(vm, factories, interrupt_fabric), + } + } +} + +fn init_vm_with( + vm: &AxVM, + factories: &axdevice::DeviceFactoryRegistry, + interrupt_fabric: crate::InterruptFabric, +) -> AxResult { + complete_vm_init(vm, interrupt_fabric, |resources, interrupt_fabric| { + let placements = vcpu_placements(resources); + let vcpus = PreparedVcpus::create(vm.id(), &placements, |_| Ok(X86VCpuCreateConfig))?; + let mut devices = PreparedDevices::build_common(resources, factories, interrupt_fabric)?; + register_arch_devices(resources.config(), &mut devices.devices)?; + devices.register_special_devices(vm)?; + validate_guest_dtb(resources)?; + + let mut owned_regions = guest_owned_regions(resources); + append_arch_owned_regions(&mut owned_regions); + map_guest_address_space(vm, resources, devices.devices(), &owned_regions)?; + map_arch_address_space(resources)?; + vcpus.setup(resources, build_vcpu_setup_config)?; + + Ok(PreparedVm::new(vcpus, devices)) + }) +} + +fn build_vcpu_setup_config( + config: &AxVMConfig, + memory_regions: &[crate::vm::VMMemoryRegion], +) -> AxResult<::SetupConfig> { + let mut setup_config = X86VCpuSetupConfig { + emulate_com1: config + .emu_devices() + .iter() + .any(|device| device.emu_type == EmulatedDeviceType::Console), + guest_memory_regions: memory_regions + .iter() + .map(|region| X86GuestMemoryRegion { + gpa: X86GuestPhysAddr::from_usize(region.gpa.as_usize()), + hva: X86HostVirtAddr::from_usize(region.hva.as_usize()), + size: region.size(), + }) + .collect(), + ..Default::default() + }; + for port in config.pass_through_ports() { + x86_result(setup_config.add_passthrough_port_range(port.base, port.length))?; + } + Ok(setup_config) +} + +fn register_arch_devices(config: &AxVMConfig, devices: &mut axdevice::AxVmDevices) -> AxResult { + for port in config.pass_through_ports() { + let passthrough = Arc::new(super::port::HostPortPassthrough::new( + port.base, + port.length, + )?); + let range = passthrough.address_range(); + debug!( + "PT port region: [{:#x}~{:#x}]", + range.start.number(), + range.end.number(), + ); + devices + .register(PortDeviceAdapter::from_arc(passthrough)) + .map_err(|err| { + ax_err_type!(InvalidInput, alloc::format!("register PT port: {err:?}")) + })?; + } + for device_config in config.emu_devices() { + super::register_arch_device(device_config, devices)?; + } + Ok(()) +} + +fn append_arch_owned_regions(regions: &mut alloc::vec::Vec) { + #[cfg(feature = "vmx")] + regions.push(GuestOwnedRegion::new( + x86_vcpu::X86_APIC_ACCESS_GPA, + PAGE_SIZE_4K, + crate::layout::VmRegionKind::Reserved, + )); + #[cfg(not(feature = "vmx"))] + let _ = regions; +} + +fn map_arch_address_space(resources: &mut AxVMResources) -> AxResult { + #[cfg(feature = "vmx")] + resources.address_space.map_linear( + axvm_types::GuestPhysAddr::from(x86_vcpu::X86_APIC_ACCESS_GPA), + x86_apic_access_page_addr(), + PAGE_SIZE_4K, + MappingFlags::DEVICE | MappingFlags::READ | MappingFlags::WRITE, + )?; + #[cfg(not(feature = "vmx"))] + let _ = resources; + Ok(()) +} + +fn guest_page_table_levels(vcpu_mappings: &[(usize, Option, usize)]) -> usize { + let mut levels = 4; + for cpu_id in crate::architecture::ops::target_phys_cpu_ids(vcpu_mappings) { + levels = levels.min(crate::percpu::cpu_max_guest_page_table_levels(cpu_id).unwrap_or(4)); + } + levels +} diff --git a/virtualization/axvm/src/architecture/capabilities.rs b/virtualization/axvm/src/architecture/capabilities.rs new file mode 100644 index 0000000000..2cd187c217 --- /dev/null +++ b/virtualization/axvm/src/architecture/capabilities.rs @@ -0,0 +1,62 @@ +//! Small capability boundaries implemented by the selected guest architecture. + +use ax_errno::AxResult; + +/// Guest firmware preparation performed before common VM memory loading. +pub(crate) trait GuestBootPlatform { + fn init_guest_boot_resources() {} + + fn prepare_guest_boot( + _vm_config: &mut crate::config::AxVMConfig, + _vm_create_config: &mut axvmconfig::AxVMCrateConfig, + _provider: &dyn crate::boot::BootImageProvider, + ) -> AxResult> { + Ok(None) + } +} + +/// Architecture-specific guest image planning layered over common byte loading. +pub(crate) trait BootImagePlatform { + fn default_boot_firmware_load_gpa( + _config: &axvmconfig::AxVMCrateConfig, + ) -> Option { + None + } + + fn load_images_from_memory( + loader: &mut crate::boot::images::ImageLoaderCore<'_>, + images: crate::boot::StaticVmImage, + ) -> AxResult { + loader.load_standard_images_from_memory(images, Self::load_guest_dtb) + } + + #[cfg(any(feature = "fs", feature = "host-fs"))] + fn load_images_from_filesystem( + loader: &mut crate::boot::images::ImageLoaderCore<'_>, + ) -> AxResult { + loader.load_standard_images_from_filesystem(Self::load_guest_dtb) + } + + fn load_guest_dtb( + _loader: &crate::boot::images::ImageLoaderCore<'_>, + _dtb: &crate::boot::fdt::GuestDtbImage, + ) -> AxResult { + Ok(()) + } + + fn is_x86_linux_image_config( + _config: &axvmconfig::AxVMCrateConfig, + _provider: &dyn crate::boot::BootImageProvider, + ) -> bool { + false + } +} + +/// Architecture-specific host timer policy used by the ArceOS adapter. +pub(crate) trait HostTimePlatform { + fn set_oneshot_timer(deadline_ns: u64) { + ax_std::os::arceos::modules::ax_hal::time::set_oneshot_timer(deadline_ns); + } + + fn register_timer_callback() {} +} diff --git a/virtualization/axvm/src/architecture/cpu_up.rs b/virtualization/axvm/src/architecture/cpu_up.rs new file mode 100644 index 0000000000..7d9e69f218 --- /dev/null +++ b/virtualization/axvm/src/architecture/cpu_up.rs @@ -0,0 +1,67 @@ +//! Shared secondary-vCPU boot flow for architectures that expose CPU-up exits. + +use ax_errno::AxResult; +use axvm_types::GuestPhysAddr; + +use crate::architecture::{ArchOps, BoundVcpuExit, VcpuRunAction}; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct CpuUpExit { + pub(crate) target_cpu: u64, + pub(crate) entry_point: GuestPhysAddr, + pub(crate) arg: u64, +} + +pub(crate) trait CpuUpOps: ArchOps { + fn set_cpu_up_success(vcpu: &crate::vm::AxVCpuRef) { + vcpu.set_gpr(0, 0); + } + + fn target_vcpu_id(vm: &crate::AxVMRef, target_cpu: u64) -> Option { + vm.get_vcpu_affinities_pcpu_ids() + .iter() + .find_map(|(vcpu_id, _, phys_id)| (*phys_id == target_cpu as usize).then_some(*vcpu_id)) + } +} + +pub(crate) fn handle( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + exit: CpuUpExit, +) -> AxResult> { + let vm_id = vm.id(); + let vcpu_id = vcpu.id(); + info!( + "VM[{vm_id}]'s VCpu[{vcpu_id}] try to boot target_cpu [{}] entry_point={:x} arg={:#x}", + exit.target_cpu, exit.entry_point, exit.arg + ); + + let Some(target_vcpu_id) = A::target_vcpu_id(vm, exit.target_cpu) else { + warn!( + "VM[{vm_id}] cannot resolve architecture CPU target {} to a VM-local vCPU", + exit.target_cpu + ); + vcpu.set_return_value(usize::MAX); + return Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })); + }; + + match crate::runtime::vcpus::vcpu_on( + vm.clone(), + target_vcpu_id, + exit.entry_point, + exit.arg as _, + ) { + Ok(()) => A::set_cpu_up_success(vcpu), + Err(err) => { + warn!("Failed to boot VM[{vm_id}] VCpu[{target_vcpu_id}]: {err:?}"); + vcpu.set_return_value(usize::MAX); + } + } + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) +} diff --git a/virtualization/axvm/src/architecture/exit.rs b/virtualization/axvm/src/architecture/exit.rs new file mode 100644 index 0000000000..2c1d95c6f6 --- /dev/null +++ b/virtualization/axvm/src/architecture/exit.rs @@ -0,0 +1,58 @@ +//! Architecture-neutral handlers for exits shared by every guest architecture. + +use ax_errno::AxResult; +use axvm_types::VmArchVcpuOps; + +use super::{ArchOps, BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; + +pub(crate) fn handle_mmio_read( + vm: &crate::AxVM, + vcpu: &crate::vm::AxVCpuRef, + exit: MmioReadExit, +) -> AxResult> { + let raw = vm.get_devices()?.handle_mmio_read(exit.addr, exit.width)?; + let masked = raw & crate::vm::width_mask(exit.width); + let val = if exit.signed_ext { + crate::vm::sign_extend_value(masked, exit.width) + } else { + masked & crate::vm::width_mask(exit.reg_width) + }; + vcpu.set_gpr(exit.reg, val); + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn handle_mmio_write( + vm: &crate::AxVMRef, + exit: MmioWriteExit, +) -> AxResult> { + vm.handle_mmio_write(exit.addr, exit.width, exit.data as usize)?; + A::after_mmio_write(vm); + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn handle_hypercall( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + exit: HypercallExit, +) -> AxResult> { + debug!("Hypercall [{:#x}] args {:x?}", exit.nr, exit.args); + match crate::runtime::hvc::HyperCall::new(vm.clone(), exit.nr, exit.args) { + Ok(hypercall) => { + let ret_val = match hypercall.execute() { + Ok(ret_val) => ret_val as isize, + Err(err) => { + warn!("Hypercall [{:#x}] failed: {err:?}", exit.nr); + -1 + } + }; + vcpu.set_return_value(ret_val as usize); + } + Err(err) => { + warn!("Hypercall [{:#x}] failed: {err:?}", exit.nr); + } + } + Ok(BoundVcpuExit::Complete(VcpuRunAction { + waits_for_event: false, + stop_reason: None, + })) +} diff --git a/virtualization/axvm/src/architecture/mod.rs b/virtualization/axvm/src/architecture/mod.rs new file mode 100644 index 0000000000..7d10f101d2 --- /dev/null +++ b/virtualization/axvm/src/architecture/mod.rs @@ -0,0 +1,11 @@ +//! Architecture-neutral contracts shared by target implementations. + +mod capabilities; +mod exit; +pub(crate) mod ops; +mod types; + +pub(crate) use capabilities::{BootImagePlatform, GuestBootPlatform, HostTimePlatform}; +pub(crate) use exit::{handle_hypercall, handle_mmio_read, handle_mmio_write}; +pub(crate) use ops::ArchOps; +pub(crate) use types::{BoundVcpuExit, HypercallExit, MmioReadExit, MmioWriteExit, VcpuRunAction}; diff --git a/virtualization/axvm/src/architecture/ops.rs b/virtualization/axvm/src/architecture/ops.rs new file mode 100644 index 0000000000..144d9522b7 --- /dev/null +++ b/virtualization/axvm/src/architecture/ops.rs @@ -0,0 +1,200 @@ +//! Core vCPU and nested-paging contract implemented by every target architecture. + +use alloc::{format, vec::Vec}; + +use ax_errno::{AxResult, ax_err}; +use ax_memory_addr::VirtAddr; +use axaddrspace::NestedPageTableOps; +use axvm_types::{VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState}; + +use super::{BoundVcpuExit, VcpuRunAction}; + +pub(crate) trait ArchOps { + type VCpu: VmArchVcpuOps; + type PerCpu: VmArchPerCpuOps; + type DeferredRunWork; + type NestedPageTable: NestedPageTableOps; + + fn has_hardware_support() -> bool; + + fn clean_dcache_range(_addr: VirtAddr, _size: usize) {} + + fn register_platform_irq_injector() {} + + fn vcpu_affinities( + cpu_num: usize, + phys_cpu_ids: Option<&[usize]>, + phys_cpu_sets: Option<&[usize]>, + ) -> Vec<(usize, Option, usize)> { + default_vcpu_affinities(cpu_num, phys_cpu_ids, phys_cpu_sets) + } + + fn set_vcpu_on_args(vcpu: &crate::vm::AxVCpuRef, _vcpu_id: usize, arg: usize) { + vcpu.set_gpr(0, arg); + } + + fn before_first_run(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) {} + + fn before_vcpu_run(_vm: &crate::AxVMRef, _vcpu: &crate::vm::AxVCpuRef) {} + + fn inject_pending_interrupt( + _vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + interrupt: crate::vm::PendingInterrupt, + ) { + match interrupt { + crate::vm::PendingInterrupt::Normal(vector) => { + trace!( + "Injecting queued interrupt {vector:#x} into VM[{}] VCpu[{}]", + vcpu.vm_id(), + vcpu.id() + ); + if let Err(err) = vcpu.inject_interrupt(vector) { + warn!( + "Failed to inject queued interrupt {vector:#x} into VM[{}] VCpu[{}]: \ + {err:?}", + vcpu.vm_id(), + vcpu.id() + ); + } + } + crate::vm::PendingInterrupt::External { + vector, + physical_irq, + } => { + warn!( + "VM[{}] VCpu[{}] dropped unsupported external interrupt vector={vector:#x}, \ + physical_irq={physical_irq:#x}", + vcpu.vm_id(), + vcpu.id() + ); + } + } + } + + fn after_external_interrupt( + _vm: &crate::AxVMRef, + _vcpu: &crate::vm::AxVCpuRef, + vector: usize, + ) { + crate::host::arceos::dispatch_host_irq(vector); + crate::check_timer_events(); + } + + fn on_last_vcpu_exit(_vm_id: usize) {} + + fn after_mmio_write(_vm: &crate::AxVMRef) {} + + fn handle_vcpu_exit_bound( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + exit: ::Exit, + ) -> AxResult>; + + fn finish_deferred_run_work( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + work: Self::DeferredRunWork, + ) -> AxResult; + + fn run_vcpu( + vm: &crate::AxVMRef, + vcpu: &crate::vm::AxVCpuRef, + ) -> AxResult + where + Self: Sized, + { + let vm_id = vm.id(); + let vcpu_id = vcpu.id(); + + match vcpu.state() { + VmVcpuState::Free => vcpu.bind()?, + VmVcpuState::Ready => {} + state => { + return ax_err!( + BadState, + format!("VCpu state is not Free or Ready, but {state:?}") + ); + } + } + + let run_result = vcpu.with_current_cpu_set(|| -> AxResult<_> { + loop { + crate::runtime::vcpus::inject_pending_interrupts::(vm.id(), vcpu_id, vcpu); + + let exit = vcpu.run()?; + trace!("{exit:#x?}"); + match Self::handle_vcpu_exit_bound(vm, vcpu, exit)? { + BoundVcpuExit::Continue => continue, + action => break Ok(action), + } + } + }); + + let unbind_result = vcpu.unbind(); + match run_result { + Ok(BoundVcpuExit::Complete(action)) => { + unbind_result?; + Ok(action) + } + Ok(BoundVcpuExit::Defer(work)) => { + unbind_result?; + Self::finish_deferred_run_work(vm, vcpu, work) + } + Ok(BoundVcpuExit::Continue) => unreachable!("continued exits do not leave run loop"), + Err(err) => { + if let Err(unbind_err) = unbind_result { + warn!( + "VM[{vm_id}] VCpu[{vcpu_id}] unbind after run error failed: {unbind_err:?}" + ); + } + Err(err) + } + } + } +} + +pub(crate) fn target_phys_cpu_ids(vcpu_mappings: &[(usize, Option, usize)]) -> Vec { + let mut cpu_ids = Vec::new(); + for (_, maybe_mask, phys_id) in vcpu_mappings { + if let Some(mask) = maybe_mask { + for cpu_id in 0..usize::BITS as usize { + if mask & (1usize << cpu_id) != 0 && !cpu_ids.contains(&cpu_id) { + cpu_ids.push(cpu_id); + } + } + } else if !cpu_ids.contains(phys_id) { + cpu_ids.push(*phys_id); + } + } + cpu_ids +} + +pub(crate) fn default_vcpu_affinities( + cpu_num: usize, + phys_cpu_ids: Option<&[usize]>, + phys_cpu_sets: Option<&[usize]>, +) -> Vec<(usize, Option, usize)> { + let mut vcpus = Vec::with_capacity(cpu_num); + for vcpu_id in 0..cpu_num { + vcpus.push((vcpu_id, None, vcpu_id)); + } + + if let Some(phys_cpu_sets) = phys_cpu_sets { + for (vcpu_id, pcpu_mask_bitmap) in phys_cpu_sets.iter().enumerate() { + if let Some(vcpu) = vcpus.get_mut(vcpu_id) { + vcpu.1 = Some(*pcpu_mask_bitmap); + } + } + } + + if let Some(phys_cpu_ids) = phys_cpu_ids { + for (vcpu_id, phys_id) in phys_cpu_ids.iter().enumerate() { + if let Some(vcpu) = vcpus.get_mut(vcpu_id) { + vcpu.2 = *phys_id; + } + } + } + + vcpus +} diff --git a/virtualization/axvm/src/architecture/sysreg.rs b/virtualization/axvm/src/architecture/sysreg.rs new file mode 100644 index 0000000000..1ab7ff7ae1 --- /dev/null +++ b/virtualization/axvm/src/architecture/sysreg.rs @@ -0,0 +1,39 @@ +//! Shared system-register device exits used by AArch64 and x86_64 guests. + +use ax_errno::AxResult; +use axvm_types::{AccessWidth, SysRegAddr, VmArchVcpuOps}; + +use crate::architecture::BoundVcpuExit; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct SysRegReadExit { + pub(crate) addr: SysRegAddr, + pub(crate) reg: usize, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct SysRegWriteExit { + pub(crate) addr: SysRegAddr, + pub(crate) value: u64, +} + +pub(crate) fn handle_read( + vm: &crate::AxVM, + vcpu: &crate::vm::AxVCpuRef, + exit: SysRegReadExit, +) -> AxResult> { + let val = vm + .get_devices()? + .handle_sys_reg_read(exit.addr, AccessWidth::Qword)?; + vcpu.set_gpr(exit.reg, val); + Ok(BoundVcpuExit::Continue) +} + +pub(crate) fn handle_write( + vm: &crate::AxVM, + exit: SysRegWriteExit, +) -> AxResult> { + vm.get_devices()? + .handle_sys_reg_write(exit.addr, AccessWidth::Qword, exit.value as usize)?; + Ok(BoundVcpuExit::Continue) +} diff --git a/virtualization/axvm/src/architecture/types.rs b/virtualization/axvm/src/architecture/types.rs new file mode 100644 index 0000000000..2bdfaf4141 --- /dev/null +++ b/virtualization/axvm/src/architecture/types.rs @@ -0,0 +1,45 @@ +//! Architecture-neutral vCPU contexts and normalized runtime actions. + +use axvm_types::{AccessWidth, GuestPhysAddr}; + +use crate::StopReason; + +/// Scheduler effects selected after an architecture-local vCPU exit. +#[derive(Debug)] +pub(crate) struct VcpuRunAction { + pub(crate) waits_for_event: bool, + pub(crate) stop_reason: Option, +} + +/// Result of handling one exit while the vCPU is still bound to the host CPU. +#[derive(Debug)] +pub(crate) enum BoundVcpuExit { + /// The exit was handled completely; re-enter the guest in the current run slice. + Continue, + /// The run slice is complete and can return this scheduler action after unbind. + Complete(VcpuRunAction), + /// Finish architecture-local work after unbinding the vCPU. + Defer(D), +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct MmioReadExit { + pub(crate) addr: GuestPhysAddr, + pub(crate) width: AccessWidth, + pub(crate) reg: usize, + pub(crate) reg_width: AccessWidth, + pub(crate) signed_ext: bool, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct MmioWriteExit { + pub(crate) addr: GuestPhysAddr, + pub(crate) width: AccessWidth, + pub(crate) data: u64, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct HypercallExit { + pub(crate) nr: u64, + pub(crate) args: [u64; 6], +} diff --git a/virtualization/axvm/src/boot/fdt/create.rs b/virtualization/axvm/src/boot/fdt/core/create.rs similarity index 70% rename from virtualization/axvm/src/boot/fdt/create.rs rename to virtualization/axvm/src/boot/fdt/core/create.rs index c9e4a25c51..d6bc933669 100644 --- a/virtualization/axvm/src/boot/fdt/create.rs +++ b/virtualization/axvm/src/boot/fdt/core/create.rs @@ -12,25 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(target_arch = "riscv64")] -use alloc::format; use alloc::{string::String, vec::Vec}; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] use core::ptr::NonNull; use ax_errno::{AxResult, ax_err_type}; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] use ax_memory_addr::MemoryAddr; use axvmconfig::AxVMCrateConfig; use fdt_edit::{Fdt, Node, NodeId}; -use super::tree::FdtTree; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -use super::tree::GuestMemorySpec; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -use crate::VMMemoryRegion; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use crate::{AxVMRef, GuestPhysAddr, boot::images::load_vm_image_from_memory}; +use super::tree::{FdtTree, GuestMemorySpec}; +use crate::{AxVMRef, GuestPhysAddr, VMMemoryRegion, boot::images::load_vm_image_from_memory}; pub fn create_guest_fdt( fdt: &Fdt, @@ -115,7 +106,12 @@ fn cpu_reg_address(fdt: &Fdt, node_id: NodeId) -> Option { .and_then(|node| node.regs().first().map(|reg| reg.address as usize)) } -fn need_cpu_node(phys_cpu_ids: &[usize], fdt: &Fdt, node_id: NodeId, node_path: &str) -> bool { +pub(crate) fn need_cpu_node( + phys_cpu_ids: &[usize], + fdt: &Fdt, + node_id: NodeId, + node_path: &str, +) -> bool { if !node_path.starts_with("/cpus/cpu@") { return true; } @@ -130,7 +126,6 @@ fn need_cpu_node(phys_cpu_ids: &[usize], fdt: &Fdt, node_id: NodeId, node_path: }) } -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] fn guest_memory_specs( new_memory: &[VMMemoryRegion], crate_config: &AxVMCrateConfig, @@ -167,71 +162,31 @@ fn guest_memory_specs( .collect() } -#[cfg(any(test, target_arch = "aarch64"))] -fn initrd_start_size_from_image_config( - ramdisk: Option<&crate::config::RamdiskInfo>, -) -> Option<(u64, u64)> { - let rd = ramdisk?; - let start = rd.load_gpa.as_usize() as u64; - let size = rd.size? as u64; - Some((start, size)) -} - #[cfg(test)] fn initrd_range_from_image_config( ramdisk: Option<&crate::config::RamdiskInfo>, ) -> Option<(u64, u64)> { - let (start, size) = initrd_start_size_from_image_config(ramdisk)?; + let ramdisk = ramdisk?; + let start = ramdisk.load_gpa.as_usize() as u64; + let size = ramdisk.size? as u64; Some((start, start.saturating_add(size))) } -#[cfg(target_arch = "aarch64")] pub fn update_fdt( fdt_src: NonNull, dtb_size: usize, vm: AxVMRef, crate_config: &AxVMCrateConfig, ) -> AxResult { + let patch_runtime = super::selected_guest_fdt_policy().patch_runtime; + // SAFETY: `fdt_src` originates from `GuestDtbImage::as_bytes`, and the + // caller supplies the exact slice length while the image remains borrowed. let fdt_bytes = unsafe { core::slice::from_raw_parts(fdt_src.as_ptr(), dtb_size) }; - let initrd_start_size = vm.with_config(|config| { - initrd_start_size_from_image_config(config.image_config.ramdisk.as_ref()) - }); - let new_fdt_bytes = patch_guest_fdt_for_runtime( - fdt_bytes, - &vm.memory_regions(), - crate_config, - initrd_start_size, - true, - )?; + let new_fdt_bytes = patch_runtime(fdt_bytes, &vm, crate_config)?; load_patched_fdt(vm, new_fdt_bytes) } -#[cfg(target_arch = "riscv64")] -pub fn update_fdt( - fdt_src: NonNull, - dtb_size: usize, - vm: AxVMRef, - crate_config: &AxVMCrateConfig, -) -> AxResult { - let fdt_bytes = unsafe { core::slice::from_raw_parts(fdt_src.as_ptr(), dtb_size) }; - let host_fdt = super::try_get_host_fdt() - .map(Fdt::from_bytes) - .transpose() - .map_err(|e| { - ax_err_type!( - InvalidData, - format!("Failed to parse host FDT while updating guest FDT: {e:#?}") - ) - })?; - let new_fdt_bytes = - patch_guest_fdt_for_runtime(fdt_bytes, &vm.memory_regions(), crate_config, None, false)?; - let new_fdt_bytes = ensure_chosen_from_host(new_fdt_bytes, host_fdt.as_ref())?; - - load_patched_fdt(vm, new_fdt_bytes) -} - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] fn load_patched_fdt(vm: AxVMRef, new_fdt_bytes: Vec) -> AxResult { let dest_addr = calculate_dtb_load_addr(vm.clone(), new_fdt_bytes.len())?; debug!( @@ -243,8 +198,7 @@ fn load_patched_fdt(vm: AxVMRef, new_fdt_bytes: Vec) -> AxResult { vm.set_guest_device_tree(dest_addr, new_fdt_bytes) } -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -pub(crate) fn patch_guest_fdt_for_runtime( +pub fn patch_guest_fdt_for_runtime( fdt_bytes: &[u8], memory_regions: &[VMMemoryRegion], crate_config: &AxVMCrateConfig, @@ -263,23 +217,6 @@ pub(crate) fn patch_guest_fdt_for_runtime( Ok(tree.finish()) } -#[cfg(target_arch = "riscv64")] -fn ensure_chosen_from_host(guest_dtb: Vec, host_fdt: Option<&Fdt>) -> AxResult> { - let Some(host_fdt) = host_fdt else { - return Ok(guest_dtb); - }; - let mut guest = FdtTree::from_bytes(&guest_dtb)?; - if guest.inner().get_by_path_id("/chosen").is_some() { - return Ok(guest.finish()); - } - let Some(host_chosen) = host_fdt.get_by_path_id("/chosen") else { - return Ok(guest.finish()); - }; - guest.copy_subtree_from(host_fdt, host_chosen, guest.inner().root_id(), false)?; - Ok(guest.finish()) -} - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] pub(crate) fn calculate_dtb_load_addr(vm: AxVMRef, fdt_size: usize) -> AxResult { const MB: usize = 1024 * 1024; @@ -292,8 +229,12 @@ pub(crate) fn calculate_dtb_load_addr(vm: AxVMRef, fdt_size: usize) -> AxResult< let use_configured_dtb_addr = config.image_config.dtb_load_gpa.is_some() && !main_memory.is_identical(); - let dtb_addr = if use_configured_dtb_addr { - config.image_config.dtb_load_gpa.unwrap() + let dtb_addr = if let Some(configured) = config + .image_config + .dtb_load_gpa + .filter(|_| use_configured_dtb_addr) + { + configured } else { let main_memory_size = main_memory.size().min(512 * MB); let addr = (main_memory.gpa + main_memory_size - fdt_size).align_down(2 * MB); @@ -309,61 +250,16 @@ pub(crate) fn calculate_dtb_load_addr(vm: AxVMRef, fdt_size: usize) -> AxResult< Ok(dtb_addr) } -#[cfg(target_arch = "aarch64")] -pub fn update_cpu_node( - fdt: &Fdt, - host_fdt: Option<&Fdt>, - crate_config: &AxVMCrateConfig, -) -> AxResult> { - let Some(host_fdt) = host_fdt else { - return Ok(fdt.encode().as_ref().to_vec()); - }; - - let phys_cpu_ids = crate_config - .base - .phys_cpu_ids - .as_deref() - .ok_or_else(|| ax_err_type!(InvalidInput, "phys_cpu_ids is missing"))?; - let mut tree = FdtTree::from_fdt(fdt.clone()); - tree.inner_mut().remove_by_path("/cpus"); - - if let Some(host_cpus_id) = host_fdt.get_by_path_id("/cpus") { - let cpus_id = - tree.copy_subtree_from(host_fdt, host_cpus_id, tree.inner().root_id(), true)?; - let cpu_paths = tree - .node_paths() - .into_iter() - .filter_map(|(id, path)| { - (path.starts_with("/cpus/cpu@") - && !need_cpu_node(phys_cpu_ids, tree.inner(), id, &path)) - .then_some(path) - }) - .collect::>(); - for path in cpu_paths { - tree.inner_mut().remove_by_path(&path); - } - if let Some(cpus) = tree.inner_mut().node_mut(cpus_id) { - for prop in [ - "riscv,cbop-block-size", - "riscv,cboz-block-size", - "riscv,cbom-block-size", - ] { - cpus.remove_property(prop); - } - } - } - - Ok(tree.finish()) -} - #[cfg(test)] mod tests { use axvmconfig::AxVMCrateConfig; use fdt_edit::{Fdt, Node, Property}; use fdt_raw::RegInfo; - use super::{cpu_node_id, initrd_range_from_image_config, need_cpu_node}; - use crate::{GuestPhysAddr, boot::fdt::tree::sanitize_bootargs, config::RamdiskInfo}; + use super::{ + super::tree::sanitize_bootargs, cpu_node_id, initrd_range_from_image_config, need_cpu_node, + }; + use crate::{GuestPhysAddr, config::RamdiskInfo}; fn prop_u32(name: &str, value: u32) -> Property { let mut prop = Property::new(name, alloc::vec![]); diff --git a/virtualization/axvm/src/boot/fdt/device.rs b/virtualization/axvm/src/boot/fdt/core/device.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/device.rs rename to virtualization/axvm/src/boot/fdt/core/device.rs diff --git a/virtualization/axvm/src/boot/fdt/core/mod.rs b/virtualization/axvm/src/boot/fdt/core/mod.rs new file mode 100644 index 0000000000..5f6c8736c9 --- /dev/null +++ b/virtualization/axvm/src/boot/fdt/core/mod.rs @@ -0,0 +1,174 @@ +//! Architecture-neutral guest device-tree preparation. + +use alloc::{format, vec::Vec}; + +use ax_errno::{AxResult, ax_err_type}; +use axvmconfig::{AxVMCrateConfig, VMBootProtocol}; + +use crate::{ + boot::{BootImageProvider, fdt::GuestDtbImage}, + config::AxVMConfig, +}; + +pub(crate) mod create; +mod device; +mod parser; +mod policy; +mod print; +pub(crate) mod tree; + +#[cfg(test)] +mod tree_tests; + +pub use create::{patch_guest_fdt_for_runtime, update_fdt}; +pub use parser::*; +pub use policy::GuestFdtPolicy; + +pub fn prepare_dtb_guest( + vm_config: &mut AxVMConfig, + vm_create_config: &mut AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult> { + if vm_create_config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { + skip_guest_dtb(vm_config, vm_create_config); + return Ok(None); + } + + let host_fdt_bytes = try_get_host_fdt(); + let guest_dtb = build_guest_dtb(vm_config, vm_create_config, provider, host_fdt_bytes)?; + enrich_guest_config(vm_config, vm_create_config, guest_dtb.as_ref())?; + Ok(guest_dtb) +} + +pub(crate) fn selected_guest_fdt_policy() -> GuestFdtPolicy { + super::guest_fdt_policy() +} + +fn skip_guest_dtb(vm_config: &mut AxVMConfig, vm_create_config: &mut AxVMCrateConfig) { + info!( + "VM[{}] uses UEFI boot protocol, skipping guest DTB handling", + vm_config.id() + ); + vm_config.clear_dtb_load_gpa(); + vm_create_config.kernel.dtb_load_addr = None; +} + +fn build_guest_dtb( + vm_config: &mut AxVMConfig, + vm_create_config: &mut AxVMCrateConfig, + provider: &dyn BootImageProvider, + host_fdt_bytes: Option<&'static [u8]>, +) -> AxResult> { + let provided_dtb = get_developer_provided_dtb(vm_config, vm_create_config, provider)?; + + match (host_fdt_bytes, provided_dtb) { + (Some(host_bytes), Some(provided)) => { + let host_fdt = parse_host_fdt(host_bytes)?; + set_phys_cpu_sets(vm_config, &host_fdt, vm_create_config)?; + info!("VM[{}] found DTB, parsing...", vm_config.id()); + reserve_excluded_device_ranges(vm_config, vm_create_config, &provided)?; + update_provided_fdt(&provided, Some(host_bytes), vm_create_config) + .map(GuestDtbImage::new) + .map(Some) + } + (Some(host_bytes), None) => { + let host_fdt = parse_host_fdt(host_bytes)?; + set_phys_cpu_sets(vm_config, &host_fdt, vm_create_config)?; + info!( + "VM[{}] DTB not found, generating from the VM configuration", + vm_config.id() + ); + setup_guest_fdt_from_vmm(host_bytes, vm_config, vm_create_config) + .map(GuestDtbImage::new) + .map(Some) + } + (None, Some(provided)) => { + info!("VM[{}] found DTB, parsing...", vm_config.id()); + reserve_excluded_device_ranges(vm_config, vm_create_config, &provided)?; + update_provided_fdt(&provided, None, vm_create_config) + .map(GuestDtbImage::new) + .map(Some) + } + (None, None) => { + warn!( + "VM[{}] no guest DTB provided; continuing without generated DTB", + vm_config.id() + ); + Ok(None) + } + } +} + +fn parse_host_fdt(host_fdt_bytes: &'static [u8]) -> AxResult { + fdt_edit::Fdt::from_bytes(host_fdt_bytes) + .map_err(|err| ax_err_type!(InvalidData, format!("Failed to parse host FDT: {err:#?}"))) +} + +fn enrich_guest_config( + vm_config: &mut AxVMConfig, + vm_create_config: &mut AxVMCrateConfig, + guest_dtb: Option<&GuestDtbImage>, +) -> AxResult { + let Some(dtb) = guest_dtb.map(GuestDtbImage::as_bytes) else { + clear_unresolved_dtb_config(vm_config, vm_create_config); + return Ok(()); + }; + + parse_reserved_memory_regions(vm_create_config, dtb)?; + parse_passthrough_devices_address(vm_config, vm_create_config, dtb)?; + parse_vm_interrupt(vm_config, dtb) +} + +fn clear_unresolved_dtb_config(vm_config: &mut AxVMConfig, vm_create_config: &mut AxVMCrateConfig) { + error!( + "VM[{}] DTB not found in memory, skipping...", + vm_config.id() + ); + let unresolved_devices = vm_config + .pass_through_devices() + .iter() + .filter(|device| device.length == 0) + .cloned() + .collect::>(); + if !unresolved_devices.is_empty() { + warn!( + "VM[{}] clearing {} unresolved passthrough discovery device(s)", + vm_config.id(), + unresolved_devices.len() + ); + for device in unresolved_devices { + vm_config.remove_pass_through_device(device); + } + } + vm_config.clear_dtb_load_gpa(); + vm_create_config.kernel.dtb_load_addr = None; +} + +fn get_developer_provided_dtb( + vm_config: &AxVMConfig, + crate_config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult>> { + match crate_config.kernel.image_location.as_deref() { + Some("memory") => Ok(provider + .static_vm_images() + .iter() + .find(|image| image.id == vm_config.id()) + .and_then(|images| images.dtb) + .map(|dtb| { + info!("DTB file in memory, size: 0x{:x}", dtb.len()); + dtb.to_vec() + })), + #[cfg(any(feature = "fs", feature = "host-fs"))] + Some("fs") => crate_config + .kernel + .dtb_path + .as_deref() + .map(|path| crate::boot::images::fs::read_full_image(path, provider)) + .transpose(), + _ => ax_errno::ax_err!( + InvalidInput, + "Unsupported image_location; use \"memory\" or enable fs feature for \"fs\"" + ), + } +} diff --git a/virtualization/axvm/src/boot/fdt/parser.rs b/virtualization/axvm/src/boot/fdt/core/parser.rs similarity index 90% rename from virtualization/axvm/src/boot/fdt/parser.rs rename to virtualization/axvm/src/boot/fdt/core/parser.rs index fcfa16172a..e7dda1b467 100644 --- a/virtualization/axvm/src/boot/fdt/parser.rs +++ b/virtualization/axvm/src/boot/fdt/core/parser.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! FDT parsing and processing functionality. +//! Architecture-neutral FDT parsing and guest configuration enrichment. use alloc::{ format, @@ -27,21 +27,18 @@ use axvmconfig::{ }; use fdt_edit::{Fdt, Node, NodeType, PciRange, PciSpace}; -#[cfg(target_arch = "aarch64")] -use crate::boot::fdt::create::update_cpu_node; use crate::{MappingFlags, config::AxVMConfig}; const PAGE_SIZE_4K: usize = 0x1000; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] pub fn try_get_host_fdt() -> Option<&'static [u8]> { - let bootarg: usize = crate::host_fdt_bootarg(); + let bootarg = super::super::host_fdt_bootarg(); if bootarg == 0 { warn!("Boot argument does not contain a host FDT pointer"); return None; } - let fdt_vaddr = crate::host_phys_to_virt(bootarg.into()); + let fdt_vaddr = super::super::host_phys_to_virt(bootarg.into()); super::tree::host_fdt_bytes_from_ptr(fdt_vaddr.as_ptr()).inspect(|bytes| { trace!("Host FDT size: 0x{:x}", bytes.len()); }) @@ -584,8 +581,8 @@ pub fn parse_passthrough_devices_address( Ok(()) } -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] pub fn parse_vm_interrupt(vm_cfg: &mut AxVMConfig, dtb: &[u8]) -> AxResult { + let decode_interrupt = super::selected_guest_fdt_policy().decode_interrupt; let fdt = Fdt::from_bytes(dtb).map_err(|e| { ax_err_type!( InvalidData, @@ -610,7 +607,7 @@ pub fn parse_vm_interrupt(vm_cfg: &mut AxVMConfig, dtb: &[u8]) -> AxResult { continue; }; for interrupt in view.interrupts() { - if let Some(irq) = passthrough_irq_from_interrupt_specifier(&interrupt.specifier) { + if let Some(irq) = decode_interrupt(&interrupt.specifier) { trace!("node: {name}, passthrough interrupt id: 0x{irq:x}"); vm_cfg.add_pass_through_irq(irq); } @@ -620,58 +617,13 @@ pub fn parse_vm_interrupt(vm_cfg: &mut AxVMConfig, dtb: &[u8]) -> AxResult { Ok(()) } -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -fn passthrough_irq_from_interrupt_specifier(specifier: &[u32]) -> Option { - #[cfg(target_arch = "aarch64")] - { - aarch64_gic_spi_from_interrupt_specifier(specifier) - } - #[cfg(target_arch = "riscv64")] - { - riscv_plic_source_from_interrupt_specifier(specifier) - } -} - -#[cfg(target_arch = "aarch64")] -fn aarch64_gic_spi_from_interrupt_specifier(specifier: &[u32]) -> Option { - (specifier.first().copied() == Some(0)) - .then(|| specifier.get(1).copied()) - .flatten() -} - -#[cfg(any(target_arch = "riscv64", test))] -fn riscv_plic_source_from_interrupt_specifier(specifier: &[u32]) -> Option { - specifier.first().copied().filter(|source| *source != 0) -} - -#[cfg(target_arch = "riscv64")] -pub fn update_provided_fdt( - provided_dtb: &[u8], - _host_dtb: Option<&[u8]>, - _crate_config: &AxVMCrateConfig, -) -> AxResult> { - Ok(provided_dtb.to_vec()) -} - -#[cfg(target_arch = "aarch64")] pub fn update_provided_fdt( provided_dtb: &[u8], host_dtb: Option<&[u8]>, crate_config: &AxVMCrateConfig, ) -> AxResult> { - let provided_fdt = Fdt::from_bytes(provided_dtb).map_err(|e| { - ax_err_type!( - InvalidData, - format!("Failed to parse provided DTB image: {e:#?}") - ) - })?; - let host_fdt = host_dtb.map(Fdt::from_bytes).transpose().map_err(|e| { - ax_err_type!( - InvalidData, - format!("Failed to parse host DTB image: {e:#?}") - ) - })?; - update_cpu_node(&provided_fdt, host_fdt.as_ref(), crate_config) + let patch_provided = super::selected_guest_fdt_policy().patch_provided; + patch_provided(provided_dtb, host_dtb, crate_config) } #[cfg(test)] @@ -803,21 +755,4 @@ mod tests { assert_eq!(ranges[0].base_gpa, 0x1000_1000); assert_eq!(ranges[0].length, 0x1000); } - - #[test] - fn riscv_plic_interrupt_uses_first_fdt_cell() { - assert_eq!( - super::riscv_plic_source_from_interrupt_specifier(&[8]), - Some(8) - ); - } - - #[test] - fn riscv_plic_interrupt_rejects_reserved_source_zero() { - assert_eq!( - super::riscv_plic_source_from_interrupt_specifier(&[0]), - None - ); - assert_eq!(super::riscv_plic_source_from_interrupt_specifier(&[]), None); - } } diff --git a/virtualization/axvm/src/boot/fdt/core/policy.rs b/virtualization/axvm/src/boot/fdt/core/policy.rs new file mode 100644 index 0000000000..2ed33515a1 --- /dev/null +++ b/virtualization/axvm/src/boot/fdt/core/policy.rs @@ -0,0 +1,17 @@ +//! Static target policies consumed by common guest FDT operations. + +use alloc::vec::Vec; + +use ax_errno::AxResult; +use axvmconfig::AxVMCrateConfig; + +pub type RuntimeFdtPatch = fn(&[u8], &crate::AxVMRef, &AxVMCrateConfig) -> AxResult>; +pub type ProvidedFdtPatch = fn(&[u8], Option<&[u8]>, &AxVMCrateConfig) -> AxResult>; + +/// Architecture operations required by common guest FDT processing. +#[derive(Clone, Copy)] +pub struct GuestFdtPolicy { + pub patch_runtime: RuntimeFdtPatch, + pub patch_provided: ProvidedFdtPatch, + pub decode_interrupt: fn(&[u32]) -> Option, +} diff --git a/virtualization/axvm/src/boot/fdt/print.rs b/virtualization/axvm/src/boot/fdt/core/print.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/print.rs rename to virtualization/axvm/src/boot/fdt/core/print.rs diff --git a/virtualization/axvm/src/boot/fdt/tree.rs b/virtualization/axvm/src/boot/fdt/core/tree.rs similarity index 99% rename from virtualization/axvm/src/boot/fdt/tree.rs rename to virtualization/axvm/src/boot/fdt/core/tree.rs index c5a6ea1640..f47dcf63bd 100644 --- a/virtualization/axvm/src/boot/fdt/tree.rs +++ b/virtualization/axvm/src/boot/fdt/core/tree.rs @@ -39,7 +39,6 @@ impl FdtTree { &self.fdt } - #[cfg(any(test, target_arch = "aarch64"))] pub(crate) fn inner_mut(&mut self) -> &mut Fdt { &mut self.fdt } diff --git a/virtualization/axvm/src/boot/fdt/tree_tests.rs b/virtualization/axvm/src/boot/fdt/core/tree_tests.rs similarity index 100% rename from virtualization/axvm/src/boot/fdt/tree_tests.rs rename to virtualization/axvm/src/boot/fdt/core/tree_tests.rs diff --git a/virtualization/axvm/src/boot/fdt/mod.rs b/virtualization/axvm/src/boot/fdt/mod.rs index bf3e63d645..f3cc239b03 100644 --- a/virtualization/axvm/src/boot/fdt/mod.rs +++ b/virtualization/axvm/src/boot/fdt/mod.rs @@ -1,257 +1,64 @@ -// Copyright 2025 The Axvisor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//! Guest device-tree artifact and selected architecture compatibility facade. -//! FDT (Flattened Device Tree) processing module for AxVisor. -//! -//! This module provides functionality for parsing and processing device tree blobs, -//! including CPU configuration, passthrough device detection, and FDT generation. - -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -mod create; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -mod device; -#[cfg(target_arch = "loongarch64")] -pub(crate) mod loongarch64; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -mod parser; -mod print; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -mod tree; - -#[cfg(test)] -mod tree_tests; - -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use alloc::format; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] use alloc::vec::Vec; -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -use ax_errno::AxResult; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use ax_errno::ax_err_type; -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -use axvmconfig::{AxVMCrateConfig, VMBootProtocol}; -// pub use print::print_fdt; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -pub use create::update_fdt; -#[cfg(any(test, target_arch = "aarch64", target_arch = "riscv64"))] -pub use parser::*; +pub use crate::arch::fdt::*; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use crate::boot::BootImageProvider; -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -use crate::config::AxVMConfig; +#[cfg(test)] +#[path = "core/mod.rs"] +pub mod test_core; -/// Guest DTB artifact produced or patched by the monitor before AxVM owns it. -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] +/// Guest DTB artifact produced or patched before AxVM owns it. #[derive(Debug, Clone)] pub struct GuestDtbImage { bytes: Vec, } -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] impl GuestDtbImage { + /// Wraps finalized guest DTB bytes. pub fn new(bytes: Vec) -> Self { Self { bytes } } + /// Returns the encoded guest DTB. pub fn as_bytes(&self) -> &[u8] { &self.bytes } } -/// Initialize LoongArch guest firmware resource handling. -#[cfg(target_arch = "loongarch64")] -pub fn init_guest_boot_resources() { - crate::boot::guest_platform::loongarch64::init(); -} - -#[cfg(target_arch = "loongarch64")] -fn handle_uefi_fdt_operations( - vm_config: &mut AxVMConfig, - vm_create_config: &mut AxVMCrateConfig, -) -> AxResult { - crate::boot::guest_platform::loongarch64::prepare_uefi_fdt_config(vm_config, vm_create_config) +#[cfg(test)] +fn guest_fdt_policy() -> test_core::GuestFdtPolicy { + test_core::GuestFdtPolicy { + patch_runtime: test_runtime_patch, + patch_provided: test_provided_patch, + decode_interrupt: |specifier| specifier.first().copied(), + } } -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -fn handle_uefi_fdt_operations( - vm_config: &mut AxVMConfig, - vm_create_config: &mut AxVMCrateConfig, -) -> AxResult { - info!( - "VM[{}] uses UEFI boot protocol, skipping guest DTB handling", - vm_config.id() - ); - vm_config.clear_dtb_load_gpa(); - vm_create_config.kernel.dtb_load_addr = None; - Ok(()) +#[cfg(test)] +fn host_fdt_bootarg() -> usize { + 0 } -/// Prepare LoongArch guest firmware FDT facts for UEFI boot. -#[cfg(target_arch = "loongarch64")] -pub fn handle_fdt_operations( - vm_config: &mut AxVMConfig, - vm_create_config: &mut AxVMCrateConfig, -) -> AxResult { - if vm_create_config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { - handle_uefi_fdt_operations(vm_config, vm_create_config)?; - return Ok(()); - } - - ax_errno::ax_err!( - Unsupported, - "LoongArch AxVisor guests currently require UEFI boot" - ) +#[cfg(test)] +fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { + ax_memory_addr::VirtAddr::from(paddr.as_usize()) } -/// Handle all FDT-related operations for guest architectures that boot with DTB. -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -pub fn handle_fdt_operations( - vm_config: &mut AxVMConfig, - vm_create_config: &mut AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> AxResult> { - if vm_create_config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { - handle_uefi_fdt_operations(vm_config, vm_create_config)?; - return Ok(None); - } - - let host_fdt_bytes = try_get_host_fdt(); - let mut guest_dtb = None; - - if let Some(host_fdt_bytes) = host_fdt_bytes { - let host_fdt = fdt_edit::Fdt::from_bytes(host_fdt_bytes) - .map_err(|e| ax_err_type!(InvalidData, format!("Failed to parse host FDT: {e:#?}")))?; - set_phys_cpu_sets(vm_config, &host_fdt, vm_create_config)?; - - if let Some(provided_dtb) = - get_developer_provided_dtb(vm_config, vm_create_config, provider)? - { - info!("VM[{}] found DTB , parsing...", vm_config.id()); - reserve_excluded_device_ranges(vm_config, vm_create_config, &provided_dtb)?; - guest_dtb = Some(GuestDtbImage::new(update_provided_fdt( - &provided_dtb, - Some(host_fdt_bytes), - vm_create_config, - )?)); - } else { - info!( - "VM[{}] DTB not found, generating based on the configuration file.", - vm_config.id() - ); - guest_dtb = Some(GuestDtbImage::new(setup_guest_fdt_from_vmm( - host_fdt_bytes, - vm_config, - vm_create_config, - )?)); - } - } else if let Some(provided_dtb) = - get_developer_provided_dtb(vm_config, vm_create_config, provider)? - { - info!("VM[{}] found DTB , parsing...", vm_config.id()); - reserve_excluded_device_ranges(vm_config, vm_create_config, &provided_dtb)?; - guest_dtb = Some(GuestDtbImage::new(update_provided_fdt( - &provided_dtb, - None, - vm_create_config, - )?)); - } else { - warn!( - "VM[{}] no guest DTB provided; continuing without generated DTB", - vm_config.id() - ); - } - - // Overlay VM config with the given DTB. - if let Some(dtb) = guest_dtb.as_ref().map(GuestDtbImage::as_bytes) { - parse_reserved_memory_regions(vm_create_config, dtb)?; - parse_passthrough_devices_address(vm_config, vm_create_config, dtb)?; - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - parse_vm_interrupt(vm_config, dtb)?; - } else { - error!( - "VM[{}] DTB not found in memory, skipping...", - vm_config.id() - ); - let unresolved_passthrough_devices = vm_config - .pass_through_devices() - .iter() - .filter(|device| device.length == 0) - .cloned() - .collect::>(); - if !unresolved_passthrough_devices.is_empty() { - warn!( - "VM[{}] clearing {} unresolved passthrough discovery device(s)", - vm_config.id(), - unresolved_passthrough_devices.len() - ); - for device in unresolved_passthrough_devices { - vm_config.remove_pass_through_device(device); - } - } - vm_config.clear_dtb_load_gpa(); - vm_create_config.kernel.dtb_load_addr = None; - } - Ok(guest_dtb) +#[cfg(test)] +fn test_runtime_patch( + fdt: &[u8], + _vm: &crate::AxVMRef, + _config: &axvmconfig::AxVMCrateConfig, +) -> ax_errno::AxResult> { + Ok(fdt.to_vec()) } -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -fn get_developer_provided_dtb( - vm_cfg: &AxVMConfig, - crate_config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> AxResult>> { - match crate_config.kernel.image_location.as_deref() { - Some("memory") => { - let vm_images = provider - .static_vm_images() - .iter() - .find(|&v| v.id == vm_cfg.id()); - - if let Some(dtb) = vm_images.and_then(|images| images.dtb) { - info!("DTB file in memory, size: 0x{:x}", dtb.len()); - return Ok(Some(dtb.to_vec())); - } - } - #[cfg(any(feature = "fs", feature = "host-fs"))] - Some("fs") => { - if let Some(dtb_path) = &crate_config.kernel.dtb_path { - let dtb_buffer = crate::boot::images::fs::read_full_image(dtb_path, provider)?; - info!("DTB file in fs, size: 0x{:x}", dtb_buffer.len()); - return Ok(Some(dtb_buffer)); - } - } - _ => { - return ax_errno::ax_err!( - InvalidInput, - "Unsupported image_location; use \"memory\" or enable fs feature for \"fs\"" - ); - } - } - Ok(None) +#[cfg(test)] +fn test_provided_patch( + fdt: &[u8], + _host_fdt: Option<&[u8]>, + _config: &axvmconfig::AxVMCrateConfig, +) -> ax_errno::AxResult> { + Ok(fdt.to_vec()) } diff --git a/virtualization/axvm/src/boot/guest_platform/mod.rs b/virtualization/axvm/src/boot/guest_platform/mod.rs index 80c43742c4..c8346477b4 100644 --- a/virtualization/axvm/src/boot/guest_platform/mod.rs +++ b/virtualization/axvm/src/boot/guest_platform/mod.rs @@ -1,2 +1,5 @@ -#[cfg(target_arch = "loongarch64")] -pub mod loongarch64; +//! Compatibility namespace for architecture-owned guest platform resources. + +pub mod loongarch64 { + pub use crate::arch::guest_platform::*; +} diff --git a/virtualization/axvm/src/boot/images/mod.rs b/virtualization/axvm/src/boot/images/mod.rs index 5bcc1cac7b..132250fb25 100644 --- a/virtualization/axvm/src/boot/images/mod.rs +++ b/virtualization/axvm/src/boot/images/mod.rs @@ -1,78 +1,31 @@ -// Copyright 2025 The Axvisor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//! Architecture-neutral guest image loading and source access. use alloc::format; use ax_errno::{AxResult, ax_err, ax_err_type}; use axvmconfig::AxVMCrateConfig; -#[cfg(target_arch = "x86_64")] -use axvmconfig::EmulatedDeviceType; -#[cfg(any(target_arch = "loongarch64", target_arch = "x86_64"))] -use axvmconfig::VMBootProtocol; -#[cfg(target_arch = "x86_64")] -use axvmconfig::VmMemMappingType; use byte_unit::Byte; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -use crate::boot::fdt::GuestDtbImage; -use crate::{ - AxVMRef, GuestPhysAddr, VMMemoryRegion, - boot::{BootImageProvider, StaticVmImage}, -}; +use super::{BootImageProvider, StaticVmImage}; +use crate::{AxVMRef, GuestPhysAddr, VMMemoryRegion}; mod linux; -#[cfg(target_arch = "x86_64")] -mod x86; -#[cfg(target_arch = "x86_64")] -use x86::boot_params as x86_boot_params; -#[cfg(target_arch = "x86_64")] -use x86::linux as x86_linux; -#[cfg(target_arch = "x86_64")] -use x86::linux_boot as x86_linux_boot; -#[cfg(target_arch = "x86_64")] -use x86::mptable as x86_mptable; -#[cfg(target_arch = "x86_64")] -use x86::multiboot as x86_boot; -#[cfg(target_arch = "x86_64")] +pub use crate::arch::ImageLoader; + +/// Return whether an x86 configuration selects direct Linux bzImage boot. +/// +/// This returns `false` on non-x86 targets. pub fn is_x86_linux_image_config( config: &AxVMCrateConfig, provider: &dyn BootImageProvider, ) -> bool { - if !should_direct_boot_x86_linux(config) { - return false; - } - - match config.kernel.image_location.as_deref() { - Some("memory") => with_memory_image(config, provider, detect_x86_linux_image).is_some(), - #[cfg(any(feature = "fs", feature = "host-fs"))] - Some("fs") => fs::kernel_read(config, provider, x86_linux::HEADER_READ_SIZE) - .ok() - .and_then(|data| detect_x86_linux_image(&data)) - .is_some(), - _ => false, - } + crate::arch::is_x86_linux_image_config(config, provider) } -#[cfg(target_arch = "x86_64")] +/// Return the q35 PCI INTx route reserved for the passthrough block device. pub const fn x86_qemu_passthrough_block_intx() -> (u8, u8, u8, usize) { - ( - x86_mptable::QEMU_PASSTHROUGH_BLOCK_DEVICE, - x86_mptable::QEMU_PASSTHROUGH_BLOCK_FUNCTION, - x86_mptable::QEMU_PASSTHROUGH_BLOCK_PIN, - x86_mptable::qemu_passthrough_block_gsi(), - ) + (3, 0, 1, 19) } pub fn get_image_header( @@ -83,88 +36,37 @@ pub fn get_image_header( Some("memory") => with_memory_image(config, provider, linux::Header::parse).flatten(), #[cfg(any(feature = "fs", feature = "host-fs"))] Some("fs") => { - let read_size = linux::Header::hdr_size(); - let data = fs::kernel_read(config, provider, read_size).ok()?; + let data = fs::kernel_read(config, provider, linux::Header::hdr_size()).ok()?; linux::Header::parse(&data) } _ => None, } } -fn with_memory_image( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, - func: F, -) -> Option -where - F: FnOnce(&[u8]) -> R, -{ - let vm_images = provider - .static_vm_images() - .iter() - .find(|&v| v.id == config.base.id)?; - - Some(func(vm_images.kernel)) -} - -fn memory_images_for_vm( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> AxResult { - provider - .static_vm_images() - .iter() - .copied() - .find(|&v| v.id == config.base.id) - .ok_or_else(|| { - ax_err_type!( - NotFound, - "VM images are missing; pass VM configs with AXVISOR_VM_CONFIGS" - ) - }) -} - -#[cfg(target_arch = "loongarch64")] -fn provider_firmware_image_for_vm( - config: &AxVMCrateConfig, - provider: &dyn BootImageProvider, -) -> Option<&'static [u8]> { - provider - .static_firmware_images() - .iter() - .find(|&v| v.id == config.base.id) - .map(|v| v.bios) - .flatten() +pub(crate) struct ImageLoaderCore<'a> { + pub(crate) provider: &'a dyn BootImageProvider, + pub(crate) main_memory: VMMemoryRegion, + pub(crate) vm: AxVMRef, + pub(crate) config: AxVMCrateConfig, + guest_dtb: Option, + pub(crate) kernel_load_gpa: GuestPhysAddr, + pub(crate) bios_load_gpa: Option, + pub(crate) ramdisk_load_gpa: Option, } -pub struct ImageLoader<'a> { - provider: &'a dyn BootImageProvider, - main_memory: VMMemoryRegion, - vm: AxVMRef, - config: AxVMCrateConfig, - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - guest_dtb: Option, - kernel_load_gpa: GuestPhysAddr, - bios_load_gpa: Option, - ramdisk_load_gpa: Option, -} - -impl<'a> ImageLoader<'a> { - pub fn new( +impl<'a> ImageLoaderCore<'a> { + pub(crate) fn new( main_memory: VMMemoryRegion, config: AxVMCrateConfig, vm: AxVMRef, provider: &'a dyn BootImageProvider, - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] guest_dtb: Option< - GuestDtbImage, - >, + guest_dtb: Option, ) -> Self { Self { provider, main_memory, vm, config, - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] guest_dtb, kernel_load_gpa: GuestPhysAddr::default(), bios_load_gpa: None, @@ -172,15 +74,7 @@ impl<'a> ImageLoader<'a> { } } - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - fn load_guest_dtb(&self, dtb: &GuestDtbImage) -> AxResult { - let bytes = dtb.as_bytes(); - let dtb_src = core::ptr::NonNull::new(bytes.as_ptr() as *mut u8) - .ok_or_else(|| ax_err_type!(InvalidData, "Guest DTB pointer is null"))?; - crate::boot::fdt::update_fdt(dtb_src, bytes.len(), self.vm.clone(), &self.config) - } - - pub fn load(&mut self) -> AxResult { + pub(crate) fn load(&mut self) -> AxResult { self.config.kernel.validate_boot_config()?; debug!( "Loading VM[{}] images into memory region: gpa={:#x}, hva={:#x}, size={:#}", @@ -189,17 +83,15 @@ impl<'a> ImageLoader<'a> { self.main_memory.hva, Byte::from(self.main_memory.size()) ); - - self.vm.with_config(|config| { - self.kernel_load_gpa = config.image_config.kernel_load_gpa; - self.bios_load_gpa = config.image_config.bios_load_gpa; - self.ramdisk_load_gpa = config.image_config.ramdisk.as_ref().map(|r| r.load_gpa); - }); + self.capture_prepared_load_addresses(); match self.config.kernel.image_location.as_deref() { - Some("memory") => self.load_vm_images_from_memory(), + Some("memory") => { + let images = memory_images_for_vm(&self.config, self.provider)?; + crate::arch::load_images_from_memory(self, images) + } #[cfg(any(feature = "fs", feature = "host-fs"))] - Some("fs") => fs::load_vm_images_from_filesystem(self), + Some("fs") => crate::arch::load_images_from_filesystem(self), _ => ax_err!( InvalidInput, "Unsupported image_location; use \"memory\" or enable fs feature for \"fs\"" @@ -207,477 +99,98 @@ impl<'a> ImageLoader<'a> { } } - /// Load VM images from memory - /// into the guest VM's memory space based on the VM configuration. - fn load_vm_images_from_memory(&mut self) -> AxResult { - debug!("Loading VM[{}] images from memory", self.config.base.id); - - let vm_images = memory_images_for_vm(&self.config, self.provider)?; - - #[cfg(target_arch = "x86_64")] - if should_direct_boot_x86_linux(&self.config) - && let Some(header) = detect_x86_linux_image(vm_images.kernel) - { - return self.load_x86_linux_images_from_memory( - header, - vm_images.kernel, - vm_images.ramdisk, - ); - } - - #[cfg(target_arch = "loongarch64")] - if self.loongarch_uefi_boot() { - self.load_loongarch_uefi_firmware_dtb()?; - self.add_loongarch_uefi_fw_cfg_from_memory(vm_images.kernel, vm_images.ramdisk)?; - self.load_loongarch_uefi_firmware_from_memory(vm_images.bios)?; - return Ok(()); - } - - load_vm_image_from_memory(vm_images.kernel, self.kernel_load_gpa, self.vm.clone())?; - - // Load Ramdisk image and record its size before regenerating the DTB. - if let Some(buffer) = vm_images.ramdisk { - self.load_ramdisk_from_memory(buffer)?; - } - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - { - // Load DTB image - if let Some(dtb) = &self.guest_dtb { - self.load_guest_dtb(dtb)?; - } else { - info!("VM[{}] has no guest DTB image to load", self.config.base.id); - } - } - - self.load_boot_image_from_memory(vm_images.bios)?; - - Ok(()) - } - - #[cfg(target_arch = "x86_64")] - fn load_x86_linux_images_from_memory( + pub(crate) fn load_standard_images_from_memory( &mut self, - header: x86_linux::X86LinuxHeader, - kernel: &[u8], - ramdisk: Option<&[u8]>, + images: StaticVmImage, + load_guest_dtb: fn(&Self, &crate::boot::fdt::GuestDtbImage) -> AxResult, ) -> AxResult { - self.adjust_x86_linux_dma_identity_layout()?; - let payload = x86_linux_payload(&header, kernel)?; - let initrd = if let Some(ramdisk) = ramdisk { - Some(x86_linux::X86LinuxRange::new( - self.ramdisk_load_gpa()?.as_usize(), - ramdisk.len(), - )) - } else { - None - }; - let layout = x86_linux::X86LinuxLoadLayout::new( - &header, - self.kernel_load_gpa.as_usize(), - payload.len(), - initrd, - ) - .map_err(x86_linux_layout_error)?; - - self.load_x86_linux_layout(header, layout, kernel)?; - load_vm_image_from_memory(payload, self.kernel_load_gpa, self.vm.clone())?; - - if let Some(buffer) = ramdisk { - self.load_ramdisk_from_memory(buffer)?; - } - - Ok(()) - } - - fn load_boot_image_from_memory(&self, bios: Option<&[u8]>) -> AxResult { - if !self.config.kernel.enable_bios { - return Ok(()); - } - - if let Some(buffer) = bios { - let load_gpa = self - .bios_load_gpa - .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; - load_vm_image_from_memory(buffer, load_gpa, self.vm.clone())?; - #[cfg(target_arch = "x86_64")] - if should_patch_x86_multiboot_info(&self.config) { - self.load_x86_multiboot_info(buffer, load_gpa)?; - } - return Ok(()); - } - - #[cfg(target_arch = "x86_64")] - if self.config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi { - let firmware_path = self.config.kernel.boot_firmware_path().ok_or_else(|| { - ax_errno::ax_err_type!(NotFound, "UEFI firmware image path is missed") - })?; - let load_gpa = self.bios_load_gpa.ok_or_else(|| { - ax_errno::ax_err_type!(NotFound, "UEFI firmware load addr is missed") - })?; - #[cfg(not(any(feature = "fs", feature = "host-fs")))] - let _ = (firmware_path, load_gpa); - - #[cfg(any(feature = "fs", feature = "host-fs"))] - { - info!( - "Loading UEFI firmware image {} at GPA {:#x}", - firmware_path, - load_gpa.as_usize() - ); - return fs::load_vm_image(firmware_path, load_gpa, self.vm.clone(), self.provider); - } - - #[cfg(not(any(feature = "fs", feature = "host-fs")))] - { - return Err(ax_errno::ax_err_type!( - Unsupported, - "UEFI firmware path requires the fs feature when no firmware image buffer is \ - available" - )); - } + load_vm_image_from_memory(images.kernel, self.kernel_load_gpa, self.vm.clone())?; + if let Some(ramdisk) = images.ramdisk { + self.load_ramdisk_from_memory(ramdisk)?; } - - #[cfg(target_arch = "x86_64")] - if self.should_load_default_x86_boot_image() { - let bios_load_gpa = builtin_x86_bios_load_gpa(self.bios_load_gpa)?; - info!( - "Loading built-in x86 boot image at GPA {:#x}", - bios_load_gpa.as_usize() - ); - load_vm_image_from_memory( - x86_boot::DEFAULT_BIOS_IMAGE, - bios_load_gpa, - self.vm.clone(), - )?; - #[cfg(target_arch = "x86_64")] - self.load_x86_multiboot_info(x86_boot::DEFAULT_BIOS_IMAGE, bios_load_gpa)?; + if let Some(dtb) = self.guest_dtb.as_ref() { + load_guest_dtb(self, dtb)?; } - - Ok(()) - } - - #[cfg(target_arch = "loongarch64")] - fn loongarch_uefi_boot(&self) -> bool { - self.config.kernel.effective_boot_protocol() == VMBootProtocol::Uefi + self.load_boot_image_from_memory(images.bios) } - #[cfg(target_arch = "loongarch64")] - fn add_loongarch_uefi_fw_cfg( - &self, - kernel: &'static [u8], - ramdisk: Option<&'static [u8]>, - ) -> AxResult { - let fw_cfg = crate::boot::guest_platform::loongarch64::emulated_fw_cfg(&self.config)?; - - self.vm.add_fw_cfg_device(crate::FwCfgDeviceConfig { - base: GuestPhysAddr::from(fw_cfg.base_gpa), - size: fw_cfg.length, - kernel, - initrd: ramdisk, - cmdline: self.config.kernel.cmdline.clone(), - cpu_num: self.config.base.cpu_num as u16, - platform: crate::boot::guest_platform::loongarch64::fw_cfg_platform_config( - &self.vm, - &self.config, - ), - }) - } - - #[cfg(target_arch = "loongarch64")] - fn add_loongarch_uefi_fw_cfg_from_memory( - &self, - kernel: &'static [u8], - ramdisk: Option<&'static [u8]>, + #[cfg(any(feature = "fs", feature = "host-fs"))] + pub(crate) fn load_standard_images_from_filesystem( + &mut self, + load_guest_dtb: fn(&Self, &crate::boot::fdt::GuestDtbImage) -> AxResult, ) -> AxResult { - debug!( - "VM[{}] configuring LoongArch UEFI fw_cfg payloads from memory: kernel={} bytes, \ - ramdisk={:?}", - self.config.base.id, - kernel.len(), - ramdisk.map(|data| data.len()) - ); - self.add_loongarch_uefi_fw_cfg(kernel, ramdisk) - } - - #[cfg(target_arch = "loongarch64")] - fn load_loongarch_uefi_firmware_from_memory(&self, bios: Option<&[u8]>) -> AxResult { - let firmware = bios - .or_else(|| provider_firmware_image_for_vm(&self.config, self.provider)) - .ok_or_else(|| { - ax_err_type!( - NotFound, - "LoongArch UEFI boot requires a build-time firmware image" - ) - })?; - self.load_loongarch_uefi_firmware_image(firmware) - } - - #[cfg(target_arch = "loongarch64")] - fn load_loongarch_uefi_firmware_image(&self, firmware: &[u8]) -> AxResult { - let load_gpa = self - .bios_load_gpa - .ok_or_else(|| ax_err_type!(NotFound, "LoongArch UEFI firmware load addr is missed"))?; - let flash_len = self - .config - .kernel - .memory_regions - .iter() - .find(|region| region.gpa == load_gpa.as_usize()) - .map_or(firmware.len(), |region| region.size); - fill_vm_region(load_gpa, flash_len, 0xff, self.vm.clone())?; - load_vm_image_from_memory(firmware, load_gpa, self.vm.clone()) - } - - #[cfg(target_arch = "loongarch64")] - fn load_loongarch_uefi_firmware_dtb(&self) -> AxResult { - crate::boot::guest_platform::loongarch64::prepare_uefi_runtime_config( - &self.vm, - &self.config, - ); - crate::boot::guest_platform::loongarch64::load_firmware_fdt(&self.vm, &self.config) - } - - #[cfg(target_arch = "x86_64")] - fn should_load_default_x86_boot_image(&self) -> bool { - self.config.kernel.enable_bios - && self.config.kernel.boot_firmware_path().is_none() - && self.config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot - } - - #[cfg(target_arch = "x86_64")] - fn load_x86_multiboot_info(&self, bios_image: &[u8], bios_load_gpa: GuestPhysAddr) -> AxResult { - const MULTIBOOT_INFO_GPA: usize = 0x6000; - const MULTIBOOT_MMAP_GPA: usize = 0x6040; - const MULTIBOOT_INFO_FLAGS: u32 = (1 << 0) | (1 << 6); - const MULTIBOOT_MEMORY_AVAILABLE: u32 = 1; - - let mem_base = self.main_memory.gpa.as_usize() as u64; - let mem_size = self.main_memory.size() as u64; - let mem_upper_kb = mem_size.saturating_sub(0x100000) / 1024; - - let mut mbi = [0u8; 52]; - write_u32(&mut mbi, 0, MULTIBOOT_INFO_FLAGS); - write_u32(&mut mbi, 4, 639); - write_u32(&mut mbi, 8, mem_upper_kb as u32); - write_u32(&mut mbi, 44, 24); - write_u32(&mut mbi, 48, MULTIBOOT_MMAP_GPA as u32); - - let mut mmap = [0u8; 24]; - write_u32(&mut mmap, 0, 20); - write_u64(&mut mmap, 4, mem_base); - write_u64(&mut mmap, 12, mem_size); - write_u32(&mut mmap, 20, MULTIBOOT_MEMORY_AVAILABLE); - - let mbi_gpa = (MULTIBOOT_INFO_GPA as u32).to_le_bytes(); - validate_x86_bios_patch_region(bios_image)?; - load_vm_image_from_memory(&mbi, MULTIBOOT_INFO_GPA.into(), self.vm.clone())?; - load_vm_image_from_memory(&mmap, MULTIBOOT_MMAP_GPA.into(), self.vm.clone())?; - load_vm_image_from_memory( - &mbi_gpa, - (bios_load_gpa.as_usize() + x86_boot::AXVM_BIOS_EBX_IMM_OFFSET).into(), + fs::load_vm_image( + &self.config.kernel.kernel_path, + self.kernel_load_gpa, self.vm.clone(), + self.provider, )?; + self.load_boot_image_from_filesystem()?; + if let Some(ramdisk_path) = &self.config.kernel.ramdisk_path { + self.load_ramdisk_from_filesystem(ramdisk_path)?; + } + if let Some(dtb) = self.guest_dtb.as_ref() { + load_guest_dtb(self, dtb)?; + } Ok(()) } - fn load_ramdisk_from_memory(&self, ramdisk: &[u8]) -> AxResult { + pub(crate) fn load_ramdisk_from_memory(&self, ramdisk: &[u8]) -> AxResult { let load_gpa = self.ramdisk_load_gpa()?; - let size = ramdisk.len(); - self.vm.with_config(|config| { - if let Some(ref mut rd) = config.image_config.ramdisk { - rd.size = Some(size); - } - }); + self.record_ramdisk_size(ramdisk.len()); info!( "Loading ramdisk image from memory ({} bytes) into GPA @{:#x}", - size, + ramdisk.len(), load_gpa.as_usize() ); load_vm_image_from_memory(ramdisk, load_gpa, self.vm.clone()) } - fn ramdisk_load_gpa(&self) -> AxResult { + pub(crate) fn ramdisk_load_gpa(&self) -> AxResult { self.ramdisk_load_gpa - .ok_or_else(|| ax_errno::ax_err_type!(NotFound, "Ramdisk load addr is missed")) - } - - #[cfg(target_arch = "x86_64")] - fn adjust_x86_linux_dma_identity_layout(&mut self) -> AxResult { - if !self.main_memory.is_identical() { - return Ok(()); - } - - let memory_base = self.main_memory.gpa.as_usize(); - let configured_kernel = self.config.kernel.kernel_load_addr; - let configured_ramdisk = self.config.kernel.ramdisk_load_addr; - - self.kernel_load_gpa = GuestPhysAddr::from(memory_base + configured_kernel); - if let Some(ramdisk_load_addr) = configured_ramdisk { - self.ramdisk_load_gpa = Some(GuestPhysAddr::from(memory_base + ramdisk_load_addr)); - } - - self.vm.with_config(|config| { - config.image_config.kernel_load_gpa = self.kernel_load_gpa; - if let Some(load_gpa) = self.ramdisk_load_gpa - && let Some(ref mut ramdisk) = config.image_config.ramdisk - { - ramdisk.load_gpa = load_gpa; - } - }); - - info!( - "Adjusted x86 Linux identity DMA layout for VM[{}]: memory_base={:#x}, \ - kernel_load_gpa={:#x}, ramdisk_load_gpa={:?}", - self.vm.id(), - memory_base, - self.kernel_load_gpa.as_usize(), - self.ramdisk_load_gpa - ); - Ok(()) - } - - #[cfg(target_arch = "x86_64")] - fn load_x86_linux_layout( - &self, - header: x86_linux::X86LinuxHeader, - layout: x86_linux::X86LinuxLoadLayout, - kernel: &[u8], - ) -> AxResult { - info!( - "x86 Linux layout for VM[{}]: header={:#x?}, payload_offset={:#x}, \ - boot_params=[{:#x}..{:#x}), boot_stub=[{:#x}..{:#x}), kernel=[{:#x}..{:#x}), \ - initrd={:?}", - self.config.base.id, - header, - header.payload_offset(), - layout.boot_params.start, - layout.boot_params.end().unwrap(), - layout.boot_stub.start, - layout.boot_stub.end().unwrap(), - layout.kernel.start, - layout.kernel.end().unwrap(), - layout.initrd - ); - - let boot_params = self.build_x86_boot_params(header, layout, kernel)?; - let boot_stub = self.build_x86_linux_boot_stub(&layout)?; - let mp_table = x86_mptable::build(); - load_vm_image_from_memory( - &boot_params, - layout.boot_params.start.into(), - self.vm.clone(), - )?; - load_vm_image_from_memory(&boot_stub, layout.boot_stub.start.into(), self.vm.clone())?; - load_vm_image_from_memory(&mp_table, x86_mptable::MP_TABLE_GPA.into(), self.vm.clone())?; - self.install_x86_linux_boot_entry(&layout); - Ok(()) + .ok_or_else(|| ax_err_type!(NotFound, "Ramdisk load addr is missed")) } - #[cfg(target_arch = "x86_64")] - fn build_x86_linux_boot_stub( - &self, - layout: &x86_linux::X86LinuxLoadLayout, - ) -> AxResult<[u8; x86_linux::BOOT_STUB_SIZE]> { - x86_linux_boot::build_boot_image(layout).map_err(|err| { - ax_errno::ax_err_type!( - InvalidInput, - format!("failed to build x86 Linux boot stub: {err:?}") - ) - }) - } - - #[cfg(target_arch = "x86_64")] - fn install_x86_linux_boot_entry(&self, layout: &x86_linux::X86LinuxLoadLayout) { - let entry = GuestPhysAddr::from(x86_linux_boot::DEFAULT_LINUX_BOOT_LOAD_GPA); + fn capture_prepared_load_addresses(&mut self) { self.vm.with_config(|config| { - config.cpu_config.bsp_entry = entry; - config.cpu_config.ap_entry = entry; + self.kernel_load_gpa = config.image_config.kernel_load_gpa; + self.bios_load_gpa = config.image_config.bios_load_gpa; + self.ramdisk_load_gpa = config.image_config.ramdisk.as_ref().map(|r| r.load_gpa); }); - info!( - "x86 Linux direct boot entry for VM[{}]: stub={:#x}, boot_params={:#x}, \ - kernel_entry={:#x}, initrd={:?}", - self.config.base.id, - layout.boot_stub.start, - layout.boot_params.start, - layout.kernel.start, - layout.initrd - ); } - #[cfg(target_arch = "x86_64")] - fn build_x86_boot_params( - &self, - header: x86_linux::X86LinuxHeader, - layout: x86_linux::X86LinuxLoadLayout, - kernel: &[u8], - ) -> AxResult<[u8; x86_linux::BOOT_PARAMS_SIZE]> { - let mut builder = x86_boot_params::BootParamsBuilder::new( - kernel, - header, - layout, - x86_linux::X86LinuxRange::new(self.main_memory.gpa.as_usize(), self.main_memory.size()), - ); - let command_line = self.config.kernel.cmdline.as_deref().ok_or_else(|| { - ax_errno::ax_err_type!( - InvalidInput, - "x86 Linux direct boot requires kernel.cmdline in the VM config" - ) - })?; - builder.set_command_line(command_line).map_err(|err| { - ax_errno::ax_err_type!( - InvalidInput, - format!("invalid x86 Linux command line: {err:?}") - ) - })?; - - for memory in &self.config.kernel.memory_regions { - if memory.map_type == VmMemMappingType::MapAlloc { - builder.add_ram_range(x86_linux::X86LinuxRange::new(memory.gpa, memory.size)); - } + fn load_boot_image_from_memory(&self, bios: Option<&[u8]>) -> AxResult { + if !self.config.kernel.enable_bios { + return Ok(()); } + let Some(bios) = bios else { + return Ok(()); + }; + let load_gpa = self + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; + load_vm_image_from_memory(bios, load_gpa, self.vm.clone()) + } - for device in &self.config.devices.passthrough_devices { - builder.add_reserved_range(x86_linux::X86LinuxRange::new( - device.base_gpa, - device.length, - )); - } - for address in &self.config.devices.passthrough_addresses { - builder.add_reserved_range(x86_linux::X86LinuxRange::new( - address.base_gpa, - address.length, - )); - } - for device in &self.config.devices.emu_devices { - if matches!(device.emu_type, EmulatedDeviceType::X86IoApic) { - builder.add_reserved_range(x86_linux::X86LinuxRange::new( - device.base_gpa, - device.length, - )); - } + #[cfg(any(feature = "fs", feature = "host-fs"))] + fn load_boot_image_from_filesystem(&self) -> AxResult { + if !self.config.kernel.enable_bios { + return Ok(()); } - builder.add_reserved_range(x86_mptable::reserved_range()); - - builder.build().map_err(|err| { - ax_errno::ax_err_type!( - InvalidInput, - format!("failed to build x86 boot_params: {err:?}") - ) - }) + let Some(path) = self.config.kernel.boot_firmware_path() else { + return Ok(()); + }; + let load_gpa = self + .bios_load_gpa + .ok_or_else(|| ax_err_type!(NotFound, "boot firmware load address is missing"))?; + fs::load_vm_image(path, load_gpa, self.vm.clone(), self.provider) } #[cfg(any(feature = "fs", feature = "host-fs"))] - fn load_ramdisk_from_filesystem(&self, ramdisk_path: &str) -> AxResult { - let load_gpa = self - .vm - .with_config(|config| config.image_config.ramdisk.as_ref().map(|r| r.load_gpa)) - .ok_or_else(|| ax_errno::ax_err_type!(NotFound, "Ramdisk load addr is missed"))?; + pub(crate) fn load_ramdisk_from_filesystem(&self, ramdisk_path: &str) -> AxResult { + let load_gpa = self.ramdisk_load_gpa()?; let ramdisk_size = fs::image_size(ramdisk_path, self.provider)?; - self.vm.with_config(|config| { - if let Some(ref mut rd) = config.image_config.ramdisk { - rd.size = Some(ramdisk_size); - } - }); + self.record_ramdisk_size(ramdisk_size); info!( "Loading ramdisk image from filesystem {} ({} bytes) into GPA @{:#x}", ramdisk_path, @@ -686,6 +199,46 @@ impl<'a> ImageLoader<'a> { ); fs::load_vm_image(ramdisk_path, load_gpa, self.vm.clone(), self.provider) } + + fn record_ramdisk_size(&self, size: usize) { + self.vm.with_config(|config| { + if let Some(ramdisk) = config.image_config.ramdisk.as_mut() { + ramdisk.size = Some(size); + } + }); + } +} + +fn with_memory_image( + config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, + func: F, +) -> Option +where + F: FnOnce(&[u8]) -> R, +{ + provider + .static_vm_images() + .iter() + .find(|image| image.id == config.base.id) + .map(|image| func(image.kernel)) +} + +fn memory_images_for_vm( + config: &AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult { + provider + .static_vm_images() + .iter() + .copied() + .find(|image| image.id == config.base.id) + .ok_or_else(|| { + ax_err_type!( + NotFound, + "VM images are missing; pass VM configs with AXVISOR_VM_CONFIGS" + ) + }) } pub fn load_vm_image_from_memory( @@ -694,26 +247,13 @@ pub fn load_vm_image_from_memory( vm: AxVMRef, ) -> AxResult { let mut buffer_pos = 0; - let image_size = image_buffer.len(); - - debug!( - "loading VM image from memory {:?} {}", - load_addr, - image_buffer.len() - ); - let image_load_regions = vm.get_image_load_region(load_addr, image_size)?; for region in image_load_regions { - let region_len = region.len(); - let bytes_to_write = region_len.min(image_size - buffer_pos); - - // SAFETY: `region` is valid writable guest memory obtained from - // `vm.get_image_load_region()`; `bytes_to_write <= region.len()` is - // guaranteed by `region_len.min(image_size - buffer_pos)`; and - // `image_buffer[buffer_pos..]` has at least `bytes_to_write` bytes. - // The source and destination do not overlap (guest HPA vs host image buffer). + let bytes_to_write = region.len().min(image_size - buffer_pos); + // SAFETY: The destination comes from `get_image_load_region`, the source + // contains `bytes_to_write` bytes, and guest memory cannot overlap the image. unsafe { core::ptr::copy_nonoverlapping( image_buffer[buffer_pos..].as_ptr(), @@ -721,15 +261,9 @@ pub fn load_vm_image_from_memory( bytes_to_write, ); } - crate::clean_dcache_range((region.as_ptr() as usize).into(), bytes_to_write); - - // Update the position of the buffer. buffer_pos += bytes_to_write; - - // If the buffer is fully written, exit the loop. - if buffer_pos >= image_size { - debug!("copy size: {bytes_to_write}"); + if buffer_pos == image_size { break; } } @@ -744,225 +278,21 @@ pub fn load_vm_image_from_memory( } } -#[cfg(target_arch = "loongarch64")] -fn fill_vm_region(load_addr: GuestPhysAddr, size: usize, byte: u8, vm: AxVMRef) -> AxResult { - let image_load_regions = vm.get_image_load_region(load_addr, size)?; - let mut filled_size = 0; - - for region in image_load_regions { - unsafe { - core::ptr::write_bytes(region.as_mut_ptr(), byte, region.len()); - } - crate::clean_dcache_range((region.as_ptr() as usize).into(), region.len()); - filled_size += region.len(); - } - - if filled_size == size { - Ok(()) - } else { - ax_err!( - InvalidData, - format!("VM memory was only partially filled: {filled_size}/{size} bytes") - ) - } -} - #[cfg(any(feature = "fs", feature = "host-fs"))] pub mod fs { - #[cfg(target_arch = "loongarch64")] - use alloc::boxed::Box; - use alloc::vec::Vec; + use alloc::{format, vec::Vec}; - use ax_errno::{AxResult, ax_err, ax_err_type}; + use ax_errno::{AxResult, ax_err_type}; + use axvmconfig::AxVMCrateConfig; - use super::*; + use crate::{AxVMRef, GuestPhysAddr, boot::BootImageProvider}; pub fn kernel_read( config: &AxVMCrateConfig, provider: &dyn BootImageProvider, read_size: usize, ) -> AxResult> { - let file_name = &config.kernel.kernel_path; - provider.read_file_exact(file_name, read_size) - } - - /// Loads the VM image files from the filesystem - /// into the guest VM's memory space based on the VM configuration. - pub(crate) fn load_vm_images_from_filesystem(loader: &mut ImageLoader) -> AxResult { - info!("Loading VM images from filesystem"); - #[cfg(target_arch = "x86_64")] - { - if should_direct_boot_x86_linux(&loader.config) { - let kernel_probe = - kernel_read(&loader.config, loader.provider, x86_linux::HEADER_READ_SIZE); - match kernel_probe { - Ok(data) => { - if let Some(header) = detect_x86_linux_image(&data) { - let kernel = read_image_file( - &loader.config.kernel.kernel_path, - loader.provider, - )?; - return loader.load_x86_linux_images_from_filesystem(header, &kernel); - } - } - Err(err) => debug!("Unable to probe x86 Linux bzImage header: {err:?}"), - } - } - } - // Load kernel image. - #[cfg(target_arch = "loongarch64")] - if loader.loongarch_uefi_boot() { - loader.load_loongarch_uefi_firmware_dtb()?; - loader.add_loongarch_uefi_fw_cfg_from_filesystem()?; - loader.load_loongarch_uefi_firmware_from_filesystem()?; - return Ok(()); - } else { - load_vm_image( - &loader.config.kernel.kernel_path, - loader.kernel_load_gpa, - loader.vm.clone(), - loader.provider, - )?; - } - #[cfg(not(target_arch = "loongarch64"))] - load_vm_image( - &loader.config.kernel.kernel_path, - loader.kernel_load_gpa, - loader.vm.clone(), - loader.provider, - )?; - // Load boot firmware image if needed. - if loader.config.kernel.enable_bios - && let Some(bios_path) = loader.config.kernel.boot_firmware_path() - { - if let Some(bios_load_addr) = loader.bios_load_gpa { - #[cfg(target_arch = "x86_64")] - { - if should_patch_x86_multiboot_info(&loader.config) { - let bios_image = read_image_file(bios_path, loader.provider)?; - validate_x86_bios_patch_region(&bios_image)?; - load_vm_image_from_memory(&bios_image, bios_load_addr, loader.vm.clone())?; - loader.load_x86_multiboot_info(&bios_image, bios_load_addr)?; - } else { - load_vm_image( - bios_path, - bios_load_addr, - loader.vm.clone(), - loader.provider, - )?; - } - } - #[cfg(not(target_arch = "x86_64"))] - load_vm_image( - bios_path, - bios_load_addr, - loader.vm.clone(), - loader.provider, - )?; - } else { - return ax_err!(NotFound, "boot firmware load addr is missed"); - } - }; - #[cfg(target_arch = "x86_64")] - if loader.should_load_default_x86_boot_image() { - let bios_load_gpa = builtin_x86_bios_load_gpa(loader.bios_load_gpa)?; - info!( - "Loading built-in x86 boot image at GPA {:#x}", - bios_load_gpa.as_usize() - ); - load_vm_image_from_memory( - x86_boot::DEFAULT_BIOS_IMAGE, - bios_load_gpa, - loader.vm.clone(), - )?; - #[cfg(target_arch = "x86_64")] - loader.load_x86_multiboot_info(x86_boot::DEFAULT_BIOS_IMAGE, bios_load_gpa)?; - } - - // Load Ramdisk image if needed. - if let Some(ramdisk_path) = &loader.config.kernel.ramdisk_path { - loader.load_ramdisk_from_filesystem(ramdisk_path)?; - }; - #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] - { - // Load DTB image if needed. - if let Some(dtb) = &loader.guest_dtb { - loader.load_guest_dtb(dtb)?; - } - } - - Ok(()) - } - - #[cfg(target_arch = "loongarch64")] - impl ImageLoader<'_> { - fn add_loongarch_uefi_fw_cfg_from_filesystem(&self) -> AxResult { - let kernel = read_full_image(&self.config.kernel.kernel_path, self.provider)?; - let kernel: &'static [u8] = Box::leak(kernel.into_boxed_slice()); - let ramdisk = if let Some(path) = &self.config.kernel.ramdisk_path { - let ramdisk = read_full_image(path, self.provider)?; - Some(Box::leak(ramdisk.into_boxed_slice()) as &'static [u8]) - } else { - None - }; - - debug!( - "VM[{}] configuring LoongArch UEFI fw_cfg payloads from filesystem: kernel={} \ - bytes, ramdisk={:?}", - self.config.base.id, - kernel.len(), - ramdisk.map(|data| data.len()) - ); - self.add_loongarch_uefi_fw_cfg(kernel, ramdisk) - } - - fn load_loongarch_uefi_firmware_from_filesystem(&self) -> AxResult { - let firmware = - provider_firmware_image_for_vm(&self.config, self.provider).ok_or_else(|| { - ax_err_type!( - NotFound, - "LoongArch UEFI boot requires a build-time firmware image" - ) - })?; - self.load_loongarch_uefi_firmware_image(firmware) - } - } - - #[cfg(target_arch = "x86_64")] - impl ImageLoader<'_> { - fn load_x86_linux_images_from_filesystem( - &mut self, - header: x86_linux::X86LinuxHeader, - kernel: &[u8], - ) -> AxResult { - self.adjust_x86_linux_dma_identity_layout()?; - let payload = x86_linux_payload(&header, kernel)?; - let initrd = if let Some(ramdisk_path) = &self.config.kernel.ramdisk_path { - let ramdisk_size = image_size(ramdisk_path, self.provider)?; - Some(x86_linux::X86LinuxRange::new( - self.ramdisk_load_gpa()?.as_usize(), - ramdisk_size, - )) - } else { - None - }; - let layout = x86_linux::X86LinuxLoadLayout::new( - &header, - self.kernel_load_gpa.as_usize(), - payload.len(), - initrd, - ) - .map_err(x86_linux_layout_error)?; - - self.load_x86_linux_layout(header, layout, kernel)?; - load_vm_image_from_memory(payload, self.kernel_load_gpa, self.vm.clone())?; - - if let Some(ramdisk_path) = &self.config.kernel.ramdisk_path { - self.load_ramdisk_from_filesystem(ramdisk_path)?; - } - - Ok(()) - } + provider.read_file_exact(&config.kernel.kernel_path, read_size) } pub(crate) fn load_vm_image( @@ -972,214 +302,28 @@ pub mod fs { provider: &dyn BootImageProvider, ) -> AxResult { let image = provider.read_file(image_path)?; - let image_size = image.len(); - - let image_load_regions = vm.get_image_load_region(image_load_gpa, image_size)?; + let image_load_regions = vm.get_image_load_region(image_load_gpa, image.len())?; let mut offset = 0; - for buffer in image_load_regions { let end = offset + buffer.len(); let data = image.get(offset..end).ok_or_else(|| { ax_err_type!( InvalidData, - format!("Image {} has an invalid load region layout", image_path) + format!("Image {image_path} has an invalid load region layout") ) })?; buffer.copy_from_slice(data); offset = end; - crate::clean_dcache_range((buffer.as_ptr() as usize).into(), buffer.len()); } - Ok(()) } - #[cfg(target_arch = "x86_64")] - fn read_image_file(image_path: &str, provider: &dyn BootImageProvider) -> AxResult> { - provider.read_file(image_path) - } - pub fn image_size(file_name: &str, provider: &dyn BootImageProvider) -> AxResult { provider.file_size(file_name) } - #[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" - ))] pub fn read_full_image(file_name: &str, provider: &dyn BootImageProvider) -> AxResult> { provider.read_file(file_name) } } - -#[cfg(target_arch = "x86_64")] -fn should_patch_x86_multiboot_info(config: &AxVMCrateConfig) -> bool { - config.kernel.effective_boot_protocol() == VMBootProtocol::Multiboot -} - -#[cfg(target_arch = "x86_64")] -fn should_direct_boot_x86_linux(config: &AxVMCrateConfig) -> bool { - !config.kernel.enable_bios && config.kernel.effective_boot_protocol() == VMBootProtocol::Direct -} - -#[cfg(target_arch = "x86_64")] -fn detect_x86_linux_image(image: &[u8]) -> Option { - match x86_linux::X86LinuxHeader::parse(image) { - Ok(header) => Some(header), - Err(err) => { - debug!("Not an x86 Linux bzImage: {err:?}"); - None - } - } -} - -#[cfg(target_arch = "x86_64")] -fn x86_linux_payload<'a>( - header: &x86_linux::X86LinuxHeader, - image: &'a [u8], -) -> AxResult<&'a [u8]> { - let payload_offset = header.payload_offset(); - image.get(payload_offset..).ok_or_else(|| { - ax_errno::ax_err_type!( - InvalidInput, - format!( - "x86 Linux bzImage payload offset {:#x} exceeds image size {:#x}", - payload_offset, - image.len() - ) - ) - }) -} - -#[cfg(target_arch = "x86_64")] -fn x86_linux_layout_error(err: x86_linux::X86LinuxLayoutError) -> ax_errno::AxError { - ax_errno::ax_err_type!( - InvalidInput, - format!("invalid x86 Linux memory layout: {err:?}") - ) -} - -#[cfg(target_arch = "x86_64")] -fn builtin_x86_bios_load_gpa(configured_gpa: Option) -> AxResult { - let default_gpa = GuestPhysAddr::from(x86_boot::DEFAULT_BIOS_LOAD_GPA); - match configured_gpa { - Some(gpa) if gpa != default_gpa => Err(ax_errno::ax_err_type!( - InvalidInput, - format!( - "built-in x86 BIOS must be loaded at GPA {:#x}, but bios_load_addr is {:#x}; set \ - bios_path to use a relocatable external BIOS image", - default_gpa.as_usize(), - gpa.as_usize() - ) - )), - Some(gpa) => Ok(gpa), - None => Ok(default_gpa), - } -} - -#[cfg(target_arch = "x86_64")] -fn validate_x86_bios_patch_region(bios_image: &[u8]) -> AxResult { - let patch_end = x86_boot::AXVM_BIOS_EBX_IMM_OFFSET + core::mem::size_of::(); - if bios_image.len() < patch_end { - return Err(ax_errno::ax_err_type!( - InvalidInput, - format!( - "x86 BIOS image is too small for multiboot info patch: size {}, need at least {} \ - bytes for EBX immediate at offset {:#x}", - bios_image.len(), - patch_end, - x86_boot::AXVM_BIOS_EBX_IMM_OFFSET - ) - )); - } - - if bios_image[x86_boot::AXVM_BIOS_EBX_IMM_OFFSET - 1] != x86_boot::MOV_EBX_IMM32_OPCODE { - return Err(ax_errno::ax_err_type!( - InvalidInput, - format!( - "x86 BIOS image does not match axvm-bios layout: expected mov ebx, imm32 opcode \ - at offset {:#x}", - x86_boot::AXVM_BIOS_EBX_IMM_OFFSET - 1 - ) - )); - } - - Ok(()) -} - -#[cfg(target_arch = "x86_64")] -fn write_u32(buffer: &mut [u8], offset: usize, value: u32) { - buffer[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); -} - -#[cfg(target_arch = "x86_64")] -fn write_u64(buffer: &mut [u8], offset: usize, value: u64) { - buffer[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); -} - -#[cfg(all(test, target_arch = "x86_64"))] -mod tests { - use super::*; - - #[test] - fn built_in_x86_bios_uses_default_gpa_when_unspecified() { - assert_eq!( - builtin_x86_bios_load_gpa(None).unwrap(), - GuestPhysAddr::from(x86_boot::DEFAULT_BIOS_LOAD_GPA) - ); - } - - #[test] - fn built_in_x86_bios_accepts_explicit_default_gpa() { - let default_gpa = GuestPhysAddr::from(x86_boot::DEFAULT_BIOS_LOAD_GPA); - - assert_eq!( - builtin_x86_bios_load_gpa(Some(default_gpa)).unwrap(), - default_gpa - ); - } - - #[test] - fn built_in_x86_bios_rejects_non_default_gpa() { - let invalid_gpa = GuestPhysAddr::from(x86_boot::DEFAULT_BIOS_LOAD_GPA + 0x1000); - - assert!(builtin_x86_bios_load_gpa(Some(invalid_gpa)).is_err()); - } - - #[test] - fn legacy_x86_bios_config_uses_multiboot_patch() { - let mut cfg = AxVMCrateConfig::default(); - cfg.kernel.enable_bios = true; - - assert!(should_patch_x86_multiboot_info(&cfg)); - } - - #[test] - fn x86_uefi_config_skips_multiboot_patch() { - let mut cfg = AxVMCrateConfig::default(); - cfg.kernel.enable_bios = true; - cfg.kernel.boot_protocol = Some(VMBootProtocol::Uefi); - - assert!(!should_patch_x86_multiboot_info(&cfg)); - } - - #[test] - fn x86_linux_direct_boot_requires_direct_protocol() { - let mut cfg = AxVMCrateConfig::default(); - - assert!(should_direct_boot_x86_linux(&cfg)); - - cfg.kernel.enable_bios = true; - assert!(!should_direct_boot_x86_linux(&cfg)); - - cfg.kernel.boot_protocol = Some(VMBootProtocol::Uefi); - assert!(!should_direct_boot_x86_linux(&cfg)); - - cfg.kernel.boot_protocol = Some(VMBootProtocol::Direct); - assert!(!should_direct_boot_x86_linux(&cfg)); - - cfg.kernel.enable_bios = false; - assert!(should_direct_boot_x86_linux(&cfg)); - } -} diff --git a/virtualization/axvm/src/boot/images/x86/mod.rs b/virtualization/axvm/src/boot/images/x86/mod.rs deleted file mode 100644 index 5aa554161c..0000000000 --- a/virtualization/axvm/src/boot/images/x86/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! x86 image loading helpers. - -pub mod linux; -pub mod linux_boot; -pub mod mptable; -pub mod multiboot; - -pub mod boot_params; diff --git a/virtualization/axvm/src/boot/mod.rs b/virtualization/axvm/src/boot/mod.rs index 0a8c52217f..67379dc316 100644 --- a/virtualization/axvm/src/boot/mod.rs +++ b/virtualization/axvm/src/boot/mod.rs @@ -3,29 +3,23 @@ #[cfg(any(feature = "fs", feature = "host-fs"))] use ax_errno::ax_err_type; +pub mod fdt; +pub mod guest_platform; pub mod images; mod policy; +mod prepared; -#[cfg(any( - test, - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub mod fdt; -#[cfg(target_arch = "loongarch64")] -pub mod guest_platform; +pub use images::*; +pub use policy::{ + GuestAcpiTables, GuestBootDescription, GuestDeviceTree, GuestFdtBuilder, + boot_firmware_load_gpa, guest_boot_policy, +}; +pub use prepared::{PreparedGuestBoot, prepare_guest_boot}; -#[cfg(target_arch = "loongarch64")] -pub use fdt::handle_fdt_operations; -#[cfg(target_arch = "loongarch64")] -pub use fdt::init_guest_boot_resources; -#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] -pub use fdt::{GuestDtbImage, handle_fdt_operations}; -pub use images::{ImageLoader, get_image_header}; -#[cfg(target_arch = "x86_64")] -pub use images::{is_x86_linux_image_config, x86_qemu_passthrough_block_intx}; -pub use policy::{GuestAcpiTables, GuestBootDescription, GuestDeviceTree, GuestFdtBuilder}; +/// Initializes architecture-owned guest firmware resources. +pub fn init_guest_boot_resources() { + crate::arch::init_guest_boot_resources(); +} /// Build-time image bytes supplied by the hypervisor application. #[derive(Clone, Copy, Debug)] @@ -44,7 +38,6 @@ pub struct StaticVmImage { pub trait BootImageProvider { fn static_vm_images(&self) -> &'static [StaticVmImage]; - #[cfg(target_arch = "loongarch64")] fn static_firmware_images(&self) -> &'static [StaticVmImage] { &[] } diff --git a/virtualization/axvm/src/boot/policy.rs b/virtualization/axvm/src/boot/policy.rs index 14d429b3b5..305e410431 100644 --- a/virtualization/axvm/src/boot/policy.rs +++ b/virtualization/axvm/src/boot/policy.rs @@ -18,6 +18,35 @@ use alloc::vec::Vec; use axvm_types::GuestPhysAddr; +use super::BootImageProvider; + +/// Selects the guest address-adjustment policy for the current architecture. +pub fn guest_boot_policy( + config: &axvmconfig::AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> crate::config::GuestBootPolicy { + if crate::boot::is_x86_linux_image_config(config, provider) { + crate::config::GuestBootPolicy::KeepConfigured + } else { + crate::config::GuestBootPolicy::AdjustKernelForBootProtocol { + protocol: config.kernel.effective_boot_protocol(), + } + } +} + +/// Resolves the configured or architecture-default boot firmware load address. +pub fn boot_firmware_load_gpa(config: &axvmconfig::AxVMCrateConfig) -> Option { + if !config.kernel.enable_bios { + return None; + } + + config + .kernel + .bios_load_addr + .map(GuestPhysAddr::from) + .or_else(|| crate::arch::default_boot_firmware_load_gpa(config)) +} + /// Device-tree boot description owned by the VM lifecycle. #[derive(Debug, Clone)] pub struct GuestDeviceTree { diff --git a/virtualization/axvm/src/boot/prepared.rs b/virtualization/axvm/src/boot/prepared.rs new file mode 100644 index 0000000000..ca34bfd866 --- /dev/null +++ b/virtualization/axvm/src/boot/prepared.rs @@ -0,0 +1,53 @@ +//! Typed guest boot preparation shared by monitor integrations. + +use ax_errno::AxResult; +use axvmconfig::AxVMCrateConfig; + +use super::{BootImageProvider, fdt::GuestDtbImage, images::ImageLoaderCore}; +use crate::{AxVMRef, VMMemoryRegion, config::AxVMConfig}; + +/// Architecture-prepared VM configuration and optional guest DTB. +#[derive(Debug)] +pub struct PreparedGuestBoot { + config: AxVMCrateConfig, + guest_dtb: Option, +} + +impl PreparedGuestBoot { + /// Returns the architecture-enriched VM configuration. + pub const fn config(&self) -> &AxVMCrateConfig { + &self.config + } + + /// Loads all configured guest images into prepared VM memory. + /// + /// # Errors + /// + /// Returns an error when an image source is unavailable, an image layout is + /// invalid, or guest memory cannot hold the configured image. + pub fn load_images( + self, + main_memory: VMMemoryRegion, + vm: AxVMRef, + provider: &dyn BootImageProvider, + ) -> AxResult { + let mut loader = + ImageLoaderCore::new(main_memory, self.config, vm, provider, self.guest_dtb); + loader.load() + } +} + +/// Applies architecture boot preparation and returns a typed load request. +/// +/// # Errors +/// +/// Returns an error when firmware requirements are unsupported or guest boot +/// metadata cannot be parsed and validated. +pub fn prepare_guest_boot( + vm_config: &mut AxVMConfig, + mut config: AxVMCrateConfig, + provider: &dyn BootImageProvider, +) -> AxResult { + let guest_dtb = crate::arch::prepare_guest_boot(vm_config, &mut config, provider)?; + Ok(PreparedGuestBoot { config, guest_dtb }) +} diff --git a/virtualization/axvm/src/host/arceos.rs b/virtualization/axvm/src/host/arceos.rs index 15e88fda1c..cb9069dd1e 100644 --- a/virtualization/axvm/src/host/arceos.rs +++ b/virtualization/axvm/src/host/arceos.rs @@ -2,12 +2,6 @@ extern crate alloc; -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] -use alloc::boxed::Box; use core::{ sync::atomic::{AtomicUsize, Ordering}, time::Duration, @@ -21,8 +15,6 @@ use ax_std::{ }; use axvm_types::{HostPhysAddr, HostVirtAddr}; -#[cfg(target_arch = "x86_64")] -use crate::host::HostConsole; use crate::{ arch::{ArchOps, CurrentArch}, host::{HostCpu, HostMemory, HostPlatform, HostTime}, @@ -88,81 +80,19 @@ impl HostMemory for ArceOsHost { } impl HostTime for ArceOsHost { - type CancelToken = usize; - - #[cfg(target_arch = "x86_64")] - fn nanos_to_ticks(&self, nanos: u64) -> u64 { - modules::ax_hal::time::nanos_to_ticks(nanos) - } - fn monotonic_time(&self) -> Duration { modules::ax_hal::time::monotonic_time() } - #[cfg(not(target_arch = "loongarch64"))] fn set_oneshot_timer(&self, deadline_ns: u64) { - modules::ax_hal::time::set_oneshot_timer(deadline_ns); + crate::arch::set_oneshot_timer(deadline_ns); } - - #[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" - ))] - fn register_timer( - &self, - deadline_ns: u64, - callback: Box, - ) -> Self::CancelToken { - crate::timer::register_timer(deadline_ns, callback) - } - - #[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] - fn cancel_timer(&self, token: Self::CancelToken) { - crate::timer::cancel_timer(token); - } -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn monotonic_time_nanos() -> u64 { - modules::ax_hal::time::monotonic_time_nanos() -} - -#[cfg(target_arch = "aarch64")] -pub(crate) fn handle_host_irq(vector: usize) -> Option { - modules::ax_hal::irq::handle_irq(vector).then_some(vector) } pub(crate) fn dispatch_host_irq(vector: usize) { modules::ax_hal::irq::handle_irq(vector); } -#[cfg(target_arch = "loongarch64")] -pub(crate) fn set_irq_enabled(raw_irq: usize, enabled: bool) { - let gsi = match u32::try_from(raw_irq) { - Ok(gsi) => gsi, - Err(_) => { - warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: out of GSI range"); - return; - } - }; - let irq = match modules::ax_hal::irq::resolve_irq_source( - modules::ax_hal::irq::IrqSource::AcpiGsi(gsi), - ) { - Ok(irq) => irq, - Err(err) => { - warn!("failed to resolve LoongArch passthrough IRQ {raw_irq}: {err:?}"); - return; - } - }; - if let Err(err) = modules::ax_hal::irq::set_enable(irq, enabled) { - warn!( - "failed to set LoongArch passthrough IRQ {raw_irq} ({irq:?}) enabled={enabled}: \ - {err:?}" - ); - } -} - impl HostCpu for ArceOsHost { type CpuMask = api::task::AxCpuMask; @@ -232,67 +162,8 @@ fn send_ipi_to_all_except_current(cpu_num: usize) { ); } -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqContext = modules::ax_hal::irq::IrqContext; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqError = modules::ax_hal::irq::IrqError; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqHandle = modules::ax_hal::irq::IrqHandle; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqId = modules::ax_hal::irq::IrqId; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqReturn = modules::ax_hal::irq::IrqReturn; -#[cfg(target_arch = "x86_64")] -pub(crate) type ArceOsIrqSource = modules::ax_hal::irq::IrqSource; -#[cfg(target_arch = "x86_64")] -pub(crate) fn request_shared_irq( - irq: ArceOsIrqId, - handler: impl FnMut(ArceOsIrqContext) -> ArceOsIrqReturn + Send + 'static, -) -> Result { - modules::ax_hal::irq::request_shared_irq(irq, handler) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn make_irq_id(domain: u16, hwirq: u32) -> ArceOsIrqId { - modules::ax_hal::irq::IrqId::new( - modules::ax_hal::irq::IrqDomainId(domain), - modules::ax_hal::irq::HwIrq(hwirq), - ) -} - -#[cfg(all(target_arch = "x86_64", not(test)))] -pub(crate) fn set_irq_enable(irq: ArceOsIrqId, enabled: bool) -> Result<(), ArceOsIrqError> { - modules::ax_hal::irq::set_enable(irq, enabled) -} - -#[cfg(target_arch = "x86_64")] -pub(crate) fn resolve_irq_source(source: ArceOsIrqSource) -> Result { - modules::ax_hal::irq::resolve_irq_source(source) -} - -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub(crate) fn host_fdt_bootarg() -> usize { - modules::ax_hal::dtb::get_bootarg() -} - -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub(crate) fn phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { - modules::ax_hal::mem::phys_to_virt(paddr) -} - -#[cfg(all( - any(feature = "fs", feature = "host-fs"), - any(target_arch = "x86_64", target_arch = "loongarch64") -))] -pub(crate) fn shutdown_host_filesystems() -> AxResult { +#[cfg(any(feature = "fs", feature = "host-fs"))] +pub fn shutdown_host_filesystems() -> AxResult { modules::ax_fs_ng::shutdown_filesystems()?; let released = modules::ax_fs_ng::release_block_irqs_for_passthrough(); if released != 0 { @@ -301,17 +172,6 @@ pub(crate) fn shutdown_host_filesystems() -> AxResult { Ok(()) } -#[cfg(target_arch = "x86_64")] -impl HostConsole for ArceOsHost { - fn write_bytes(&self, bytes: &[u8]) { - modules::ax_hal::console::write_bytes(bytes); - } - - fn read_bytes(&self, bytes: &mut [u8]) -> usize { - modules::ax_hal::console::read_bytes(bytes) - } -} - impl HostPlatform for ArceOsHost { fn has_hardware_support(&self) -> bool { CurrentArch::has_hardware_support() diff --git a/virtualization/axvm/src/host/mod.rs b/virtualization/axvm/src/host/mod.rs index 40b161d698..f3003f1839 100644 --- a/virtualization/axvm/src/host/mod.rs +++ b/virtualization/axvm/src/host/mod.rs @@ -1,21 +1,13 @@ //! Internal host boundary used by the AxVM runtime. pub(crate) mod arceos; -#[cfg(target_arch = "aarch64")] -pub(crate) mod gic; -#[cfg(target_arch = "x86_64")] -pub(crate) mod irq; pub(crate) mod paging; pub(crate) mod task; pub(crate) mod traits; -#[cfg(target_arch = "x86_64")] -pub(crate) mod x86_port; pub(crate) fn default_host() -> &'static arceos::ArceOsHost { arceos::arceos_host() } pub(crate) use paging::PagingHandler; -#[cfg(target_arch = "x86_64")] -pub(crate) use traits::HostConsole; pub(crate) use traits::{HostCpu, HostMemory, HostPlatform, HostTime}; diff --git a/virtualization/axvm/src/host/traits.rs b/virtualization/axvm/src/host/traits.rs index 8225612376..34e9f2f687 100644 --- a/virtualization/axvm/src/host/traits.rs +++ b/virtualization/axvm/src/host/traits.rs @@ -1,13 +1,5 @@ //! Internal host capability traits used by the AxVM runtime. -extern crate alloc; - -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] -use alloc::boxed::Box; use core::time::Duration; use ax_errno::AxResult; @@ -40,35 +32,11 @@ pub trait HostMemory { /// Host time and timer operations. pub trait HostTime { - /// Timer cancellation token. - type CancelToken: Copy + Send + Sync + 'static; - - /// Convert nanoseconds to hardware ticks. - #[cfg(target_arch = "x86_64")] - fn nanos_to_ticks(&self, nanos: u64) -> u64; - /// Read monotonic host time. fn monotonic_time(&self) -> Duration; /// Program the host one-shot timer. - #[cfg(not(target_arch = "loongarch64"))] fn set_oneshot_timer(&self, deadline_ns: u64); - - /// Register a VM timer callback. - #[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" - ))] - fn register_timer( - &self, - deadline_ns: u64, - callback: Box, - ) -> Self::CancelToken; - - /// Cancel a VM timer callback. - #[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] - fn cancel_timer(&self, token: Self::CancelToken); } /// Host CPU topology and affinity operations. @@ -83,16 +51,6 @@ pub trait HostCpu { fn this_cpu_id(&self) -> usize; } -/// Host console operations. -#[cfg(target_arch = "x86_64")] -pub trait HostConsole { - /// Write raw bytes to host console. - fn write_bytes(&self, bytes: &[u8]); - - /// Read raw bytes from host console. - fn read_bytes(&self, bytes: &mut [u8]) -> usize; -} - /// Host platform lifecycle and virtualization controls. pub trait HostPlatform { /// Check whether hardware virtualization is available. diff --git a/virtualization/axvm/src/irq/mod.rs b/virtualization/axvm/src/irq/mod.rs index 0d8930aa1b..43abba80f4 100644 --- a/virtualization/axvm/src/irq/mod.rs +++ b/virtualization/axvm/src/irq/mod.rs @@ -21,11 +21,7 @@ use axdevice::IrqResolver; use axdevice_base::{InterruptTriggerMode, IrqLine, IrqLineId, IrqSink}; use axvm_types::VMInterruptMode; -#[cfg(target_arch = "riscv64")] -pub(crate) mod riscv; - /// Host platform hook for registering the RISC-V physical IRQ injector. -#[cfg(target_arch = "riscv64")] #[ax_crate_interface::def_interface] pub trait RiscvPlatformIrqInjectorIf { /// Registers a callback that forwards a physical IRQ line into the current guest. @@ -35,14 +31,20 @@ pub trait RiscvPlatformIrqInjectorIf { fn set_virtual_irq_targets(cpu_id: usize, irq_sources: &[u32]); } -#[cfg(target_arch = "riscv64")] +#[expect( + dead_code, + reason = "the RISC-V architecture backend is not compiled for this target" +)] pub(crate) fn register_riscv_virtual_irq_injector(injector: fn(usize) -> bool) { ax_crate_interface::call_interface!(RiscvPlatformIrqInjectorIf::register_virtual_irq_injector( injector )); } -#[cfg(target_arch = "riscv64")] +#[expect( + dead_code, + reason = "the RISC-V architecture backend is not compiled for this target" +)] pub(crate) fn set_riscv_virtual_irq_targets(cpu_id: usize, irq_sources: &[u32]) { ax_crate_interface::call_interface!(RiscvPlatformIrqInjectorIf::set_virtual_irq_targets( cpu_id, diff --git a/virtualization/axvm/src/lib.rs b/virtualization/axvm/src/lib.rs index 3eea2fb8bf..d1164e50d2 100644 --- a/virtualization/axvm/src/lib.rs +++ b/virtualization/axvm/src/lib.rs @@ -24,12 +24,14 @@ extern crate alloc; extern crate log; mod arch; +mod architecture; pub mod boot; mod host; pub mod irq; pub mod layout; pub mod lifecycle; mod manager; +mod npt; mod percpu; mod runtime; mod task; @@ -41,6 +43,7 @@ use crate::arch::ArchOps; pub mod config; +pub use arch::platform::*; pub use ax_cpumask::CpuMask; /// Compatibility export for legacy/common normalized VM events. /// @@ -62,11 +65,6 @@ pub use manager::{ AxvmRuntime, current_vcpu_id, current_vm_id, get_vm_by_id, get_vm_list, inject_current_vcpu_interrupt, register_vm, }; -#[cfg(target_arch = "loongarch64")] -pub use runtime::loongarch_irq::{ - register_guest_irq_route as register_loongarch_guest_irq_route, - unregister_guest_irq_routes as unregister_loongarch_guest_irq_routes, -}; pub(crate) use task::{AsVCpuTask, VCpuTask}; pub use vm::{ AxVM, AxVMRef, FwCfgDeviceConfig, PreparedMemoryLayout, VMMemoryRegion, VcpuSnapshot, @@ -84,56 +82,3 @@ pub fn check_timer_events() { pub fn clean_dcache_range(addr: ax_memory_addr::VirtAddr, size: usize) { arch::CurrentArch::clean_dcache_range(addr, size); } - -/// Return the host FDT boot argument physical address. -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub fn host_fdt_bootarg() -> usize { - host::arceos::host_fdt_bootarg() -} - -/// Convert a host physical address into a host virtual address. -#[cfg(any( - target_arch = "aarch64", - target_arch = "loongarch64", - target_arch = "riscv64" -))] -pub fn host_phys_to_virt(paddr: ax_memory_addr::PhysAddr) -> ax_memory_addr::VirtAddr { - host::arceos::phys_to_virt(paddr) -} - -/// Shut down ArceOS filesystems so guest passthrough can take ownership. -#[cfg(all( - any(feature = "fs", feature = "host-fs"), - any(target_arch = "x86_64", target_arch = "loongarch64") -))] -pub fn shutdown_host_filesystems() -> ax_errno::AxResult { - host::arceos::shutdown_host_filesystems() -} - -/// Register a native host IRQ as the source for one x86 guest IOAPIC GSI. -#[cfg(target_arch = "x86_64")] -pub fn register_x86_ioapic_irq_forwarding_route(guest_gsi: usize, host_irq: irq_framework::IrqId) { - runtime::register_x86_ioapic_irq_forwarding_route(guest_gsi, host_irq); -} - -/// Register a native host IRQ and trigger mode as the source for one x86 guest -/// IOAPIC GSI. -#[cfg(target_arch = "x86_64")] -pub fn register_x86_ioapic_irq_forwarding_route_with_trigger( - guest_gsi: usize, - host_irq: irq_framework::IrqId, - trigger: InterruptTriggerMode, -) { - runtime::register_x86_ioapic_irq_forwarding_route_with_trigger(guest_gsi, host_irq, trigger); -} - -/// Register a callback to activate one x86 guest IOAPIC GSI after the guest has -/// programmed a usable virtual IOAPIC route for it. -#[cfg(target_arch = "x86_64")] -pub fn register_x86_ioapic_irq_forwarding_activator(guest_gsi: usize, activator: fn()) { - runtime::register_x86_ioapic_irq_forwarding_activator(guest_gsi, activator); -} diff --git a/virtualization/axvm/src/lifecycle/machine.rs b/virtualization/axvm/src/lifecycle/machine.rs index cf79b8be40..79e5819a79 100644 --- a/virtualization/axvm/src/lifecycle/machine.rs +++ b/virtualization/axvm/src/lifecycle/machine.rs @@ -1,13 +1,8 @@ -use alloc::{ - boxed::Box, - string::{String, ToString}, -}; +use alloc::string::{String, ToString}; use super::{StopReason, VmLifecycleError, VmLifecycleResult, VmStatus}; -use crate::config::AxVMConfig; pub enum Machine { - Uninit(Box), Ready(R), Running { resources: R, @@ -40,7 +35,6 @@ pub enum Machine { impl Machine { pub fn status(&self) -> VmStatus { match self { - Machine::Uninit(_) => VmStatus::Uninit, Machine::Ready(_) => VmStatus::Ready, Machine::Running { .. } => VmStatus::Running, Machine::Pausing { .. } => VmStatus::Pausing, @@ -80,41 +74,6 @@ impl Machine { } } - pub fn config_mut(&mut self) -> Option<&mut AxVMConfig> { - match self { - Machine::Uninit(config) => Some(config.as_mut()), - _ => None, - } - } - - pub fn prepare_with(&mut self, f: F) -> VmLifecycleResult - where - F: FnOnce(AxVMConfig) -> VmLifecycleResult, - { - let old = core::mem::replace(self, Machine::Switching); - match old { - Machine::Uninit(config) => match f(*config) { - Ok(resources) => { - *self = Machine::Ready(resources); - Ok(()) - } - Err(err) => { - *self = Machine::Failed(err.to_string()); - Err(err) - } - }, - other => { - let from = other.status(); - *self = other; - Err(VmLifecycleError::invalid_transition( - from, - VmStatus::Ready, - "prepare", - )) - } - } - } - pub fn runtime(&self) -> Option<&H> { match self { Machine::Running { runtime, .. } @@ -579,7 +538,7 @@ impl Machine { *self = Machine::Destroyed; Ok(()) } - Machine::Uninit(_) | Machine::Failed(_) | Machine::Switching | Machine::Destroying => { + Machine::Failed(_) | Machine::Switching | Machine::Destroying => { f(None)?; *self = Machine::Destroyed; Ok(()) @@ -592,16 +551,9 @@ impl Machine { mod tests { use super::*; - fn config() -> AxVMConfig { - AxVMConfig::default_for_test(1, "lifecycle-test") - } - #[test] - fn lifecycle_allows_prepare_start_pause_resume_stop_destroy() { - let mut machine = Machine::Uninit(Box::new(config())); - assert_eq!(machine.status(), VmStatus::Uninit); - - machine.prepare_with(|_| Ok(7usize)).unwrap(); + fn lifecycle_allows_start_pause_resume_stop_destroy_from_ready() { + let mut machine = Machine::Ready(7usize); assert_eq!(machine.status(), VmStatus::Ready); machine @@ -639,19 +591,7 @@ mod tests { #[test] fn lifecycle_rejects_invalid_transitions_without_changing_state() { - let mut machine = Machine::::Uninit(Box::new(config())); - let err = machine.start_with(|_| Ok(())).unwrap_err(); - assert!(matches!( - err, - VmLifecycleError::InvalidTransition { - from: VmStatus::Uninit, - to: VmStatus::Running, - op: "start" - } - )); - assert_eq!(machine.status(), VmStatus::Uninit); - - machine.prepare_with(|_| Ok(1)).unwrap(); + let mut machine = Machine::::Ready(1); let err = machine.resume().unwrap_err(); assert!(matches!( err, diff --git a/virtualization/axvm/src/lifecycle/status.rs b/virtualization/axvm/src/lifecycle/status.rs index ab7fd12c15..9b987ee1ca 100644 --- a/virtualization/axvm/src/lifecycle/status.rs +++ b/virtualization/axvm/src/lifecycle/status.rs @@ -11,7 +11,6 @@ pub enum StopReason { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum VmStatus { - Uninit, Ready, Running, Pausing, @@ -26,7 +25,6 @@ pub enum VmStatus { impl VmStatus { pub const fn as_str(self) -> &'static str { match self { - VmStatus::Uninit => "uninit", VmStatus::Ready => "ready", VmStatus::Running => "running", VmStatus::Pausing => "pausing", @@ -45,7 +43,6 @@ impl VmStatus { pub const fn as_str_with_icon(self) -> &'static str { match self { - VmStatus::Uninit => "[..] uninit", VmStatus::Ready => "[OK] ready", VmStatus::Running => "[RUN] running", VmStatus::Pausing => "[..] pausing", diff --git a/virtualization/axvm/src/manager.rs b/virtualization/axvm/src/manager.rs index d0aa97b20f..69ae065928 100644 --- a/virtualization/axvm/src/manager.rs +++ b/virtualization/axvm/src/manager.rs @@ -54,7 +54,6 @@ pub fn get_vm_list() -> Vec { } /// Run an operation with a VM selected from the process-wide runtime registry. -#[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] pub(crate) fn with_vm(vm_id: VMId, f: F) -> Option where F: FnOnce(&AxVMRef) -> R, @@ -64,7 +63,6 @@ where } /// Return the active-vCPU mask for a VM. -#[cfg(target_arch = "x86_64")] pub(crate) fn active_vcpu_mask(vm_id: VMId) -> Option { with_vm(vm_id, |vm| { let vcpu_num = vm.vcpu_num(); @@ -77,13 +75,15 @@ pub(crate) fn active_vcpu_mask(vm_id: VMId) -> Option { } /// Inject a virtual interrupt into a VM's vCPU. -#[cfg(target_arch = "x86_64")] pub(crate) fn inject_interrupt(vm_id: VMId, vcpu_id: usize, vector: usize) -> AxResult { crate::runtime::vcpus::queue_interrupt(vm_id, vcpu_id, vector) } /// Inject a virtual interrupt into a VM's vCPU. -#[cfg(target_arch = "loongarch64")] +#[expect( + dead_code, + reason = "only the LoongArch IRQ backend injects external VM interrupts" +)] pub(crate) fn inject_vm_vcpu_interrupt(vm_id: VMId, vcpu_id: usize, vector: usize) -> AxResult { use crate::AsVCpuTask; diff --git a/virtualization/axvm/src/arch/npt.rs b/virtualization/axvm/src/npt.rs similarity index 100% rename from virtualization/axvm/src/arch/npt.rs rename to virtualization/axvm/src/npt.rs diff --git a/virtualization/axvm/src/runtime/mod.rs b/virtualization/axvm/src/runtime/mod.rs index 884c36fd93..1e30038dc8 100644 --- a/virtualization/axvm/src/runtime/mod.rs +++ b/virtualization/axvm/src/runtime/mod.rs @@ -14,18 +14,11 @@ pub(crate) mod hvc; mod ivc; - -#[cfg(target_arch = "loongarch64")] -pub mod loongarch_irq; pub(crate) mod vcpus; -#[cfg(target_arch = "x86_64")] -pub(crate) mod x86_irq; use core::sync::atomic::{AtomicUsize, Ordering}; use ax_errno::{AxResult, ax_err, ax_err_type}; -#[cfg(target_arch = "x86_64")] -use axvm_types::InterruptTriggerMode; use crate::{StopReason, VmStatus}; @@ -132,26 +125,6 @@ pub fn register_vm(vm: VMRef) -> bool { crate::manager::push_existing_vm(vm) } -/// Register a native host IRQ as the source for one x86 guest IOAPIC GSI. -#[cfg(target_arch = "x86_64")] -pub(crate) fn register_x86_ioapic_irq_forwarding_route( - guest_gsi: usize, - host_irq: irq_framework::IrqId, -) { - x86_irq::register_ioapic_irq_forwarding_route(guest_gsi, host_irq); -} - -/// Register a native host IRQ and trigger mode as the source for one x86 guest -/// IOAPIC GSI. -#[cfg(target_arch = "x86_64")] -pub(crate) fn register_x86_ioapic_irq_forwarding_route_with_trigger( - guest_gsi: usize, - host_irq: irq_framework::IrqId, - trigger: InterruptTriggerMode, -) { - x86_irq::register_ioapic_irq_forwarding_route_with_trigger(guest_gsi, host_irq, trigger); -} - #[cfg(test)] mod tests { use super::*; @@ -172,10 +145,3 @@ mod tests { } } } - -/// Register a callback to activate one x86 guest IOAPIC GSI after the guest has -/// programmed a usable virtual IOAPIC route for it. -#[cfg(target_arch = "x86_64")] -pub(crate) fn register_x86_ioapic_irq_forwarding_activator(guest_gsi: usize, activator: fn()) { - x86_irq::register_ioapic_irq_forwarding_activator(guest_gsi, activator); -} diff --git a/virtualization/axvm/src/runtime/vcpus.rs b/virtualization/axvm/src/runtime/vcpus.rs index d26a538966..dad9b3d24c 100644 --- a/virtualization/axvm/src/runtime/vcpus.rs +++ b/virtualization/axvm/src/runtime/vcpus.rs @@ -17,13 +17,11 @@ use alloc::format; use ax_errno::{AxResult, ax_err_type}; use crate::{ - AsVCpuTask, StopReason, VCpuTask, VmStatus, + AsVCpuTask, GuestPhysAddr, StopReason, VCpuTask, VmStatus, VmVcpuState, arch::{ArchOps, CurrentArch, VcpuRunAction}, runtime::{VCpuRef, VMRef, sub_running_vm_count}, vm::VmRuntimeHandle, }; -#[cfg(not(target_arch = "x86_64"))] -use crate::{GuestPhysAddr, VmVcpuState}; const KERNEL_STACK_SIZE: usize = 0x40000; // 256 KiB @@ -105,7 +103,10 @@ pub(crate) fn queue_interrupt(vm_id: usize, vcpu_id: usize, vector: usize) -> Ax Ok(()) } -#[cfg(target_arch = "loongarch64")] +#[expect( + dead_code, + reason = "only the LoongArch IRQ backend queues physical interrupts" +)] pub(crate) fn queue_external_interrupt( vm_id: usize, vcpu_id: usize, @@ -190,7 +191,10 @@ fn mark_vcpu_running(vm: &VMRef) { /// * `vcpu_id` - The ID of the VCpu to be booted. /// * `entry_point` - The entry point of the VCpu. /// * `arg` - The argument to be passed to the VCpu. -#[cfg(not(target_arch = "x86_64"))] +#[expect( + dead_code, + reason = "only non-x86 guest firmware boots secondary vCPUs" +)] pub(crate) fn vcpu_on( vm: VMRef, vcpu_id: usize, @@ -220,7 +224,10 @@ pub(crate) fn vcpu_on( Ok(()) } -#[cfg(not(target_arch = "x86_64"))] +#[expect( + dead_code, + reason = "only non-x86 guest firmware boots secondary vCPUs" +)] pub(crate) fn alloc_vcpu_task(vm: &VMRef, vcpu: VCpuRef) -> crate::AxTaskRef { crate::host::task::spawn_task(build_vcpu_task(vm, vcpu)) } @@ -308,14 +315,20 @@ fn vcpu_run() { CurrentArch::before_vcpu_run(&vm, &vcpu); match CurrentArch::run_vcpu(&vm, &vcpu) { - Ok(VcpuRunAction::Yield) => {} - Ok(VcpuRunAction::Wait) => wait(&runtime), - Ok(VcpuRunAction::Stop(reason)) => { + Ok(VcpuRunAction { + stop_reason: Some(reason), + .. + }) => { if let Err(err) = vm.stop(reason) { warn!("VM[{vm_id}] shutdown failed: {err:?}"); } notify_all_vcpus(vm_id); } + Ok(VcpuRunAction { + waits_for_event: true, + .. + }) => wait(&runtime), + Ok(VcpuRunAction { .. }) => {} Err(err) => { error!("VM[{vm_id}] run VCpu[{vcpu_id}] get error {err:?}"); if let Err(err) = vm.stop(StopReason::Fault(format!("{err:?}"))) { diff --git a/virtualization/axvm/src/timer.rs b/virtualization/axvm/src/timer.rs index da674abc30..0796a566df 100644 --- a/virtualization/axvm/src/timer.rs +++ b/virtualization/axvm/src/timer.rs @@ -3,18 +3,10 @@ extern crate alloc; use alloc::boxed::Box; -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] -use core::sync::atomic::{AtomicUsize, Ordering}; -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] -use core::time::Duration; +use core::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; use ax_kspin::SpinNoIrq; use ax_lazyinit::LazyInit; @@ -22,33 +14,19 @@ use ax_timer_list::{TimeValue, TimerEvent, TimerList}; use crate::host::{HostTime, default_host}; -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] static TOKEN: AtomicUsize = AtomicUsize::new(0); struct VmTimerEvent { - #[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] token: usize, callback: Box, } impl VmTimerEvent { - #[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" - ))] fn new(token: usize, callback: F) -> Self where F: FnOnce(TimeValue) + Send + 'static, { - #[cfg(not(any(target_arch = "x86_64", target_arch = "loongarch64")))] - let _ = token; Self { - #[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] token, callback: Box::new(callback), } @@ -64,11 +42,6 @@ impl TimerEvent for VmTimerEvent { #[ax_percpu::def_percpu] static TIMER_LIST: LazyInit>> = LazyInit::new(); -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "loongarch64" -))] pub(crate) fn register_timer( deadline_ns: u64, callback: Box, @@ -89,7 +62,6 @@ pub(crate) fn register_timer( token } -#[cfg(any(target_arch = "x86_64", target_arch = "loongarch64"))] pub(crate) fn cancel_timer(token: usize) { let next_deadline = { // SAFETY: The timer list is initialized for each CPU before VM timer @@ -120,9 +92,6 @@ pub(crate) fn check_events() { } fn rearm_host_timer(next_deadline: Option) { - #[cfg(target_arch = "loongarch64")] - let _ = next_deadline; - #[cfg(not(target_arch = "loongarch64"))] if let Some(deadline) = next_deadline { default_host().set_oneshot_timer(deadline.as_nanos() as u64); } @@ -134,6 +103,5 @@ pub(crate) fn init_percpu() { // CPU can register VM timers. let timer_list = unsafe { TIMER_LIST.current_ref_mut_raw() }; timer_list.init_once(SpinNoIrq::new(TimerList::new())); - #[cfg(target_arch = "loongarch64")] - ax_std::os::arceos::modules::ax_task::register_timer_callback(|_| check_events()); + crate::arch::register_timer_callback(); } diff --git a/virtualization/axvm/src/vcpu.rs b/virtualization/axvm/src/vcpu.rs index 7683e0f3af..626533f0c1 100644 --- a/virtualization/axvm/src/vcpu.rs +++ b/virtualization/axvm/src/vcpu.rs @@ -19,8 +19,6 @@ use core::{cell::UnsafeCell, mem::MaybeUninit}; use ax_errno::{AxResult, ax_err}; use ax_kspin::SpinNoIrq as Mutex; -#[cfg(target_arch = "x86_64")] -use axvm_types::InterruptTriggerMode; use axvm_types::{ GuestPhysAddr, NestedPagingConfig, VCpuId, VMId, VmArchPerCpuOps, VmArchVcpuOps, VmVcpuState, }; @@ -201,7 +199,10 @@ impl AxVCpu { } /// Sets the guest entry point. - #[cfg(not(target_arch = "x86_64"))] + #[expect( + dead_code, + reason = "only non-x86 guest firmware updates secondary vCPU entries" + )] pub fn set_entry(&self, entry: GuestPhysAddr) -> AxResult { self.get_arch_vcpu().set_entry(entry) } @@ -216,17 +217,6 @@ impl AxVCpu { self.get_arch_vcpu().inject_interrupt(vector) } - /// Injects an interrupt with trigger-mode metadata. - #[cfg(target_arch = "x86_64")] - pub fn inject_interrupt_with_trigger( - &self, - vector: usize, - trigger: InterruptTriggerMode, - ) -> AxResult { - self.get_arch_vcpu() - .inject_interrupt_with_trigger(vector, trigger) - } - /// Sets the guest return value. pub fn set_return_value(&self, val: usize) { self.get_arch_vcpu().set_return_value(val); diff --git a/virtualization/axvm/src/vm.rs b/virtualization/axvm/src/vm/mod.rs similarity index 94% rename from virtualization/axvm/src/vm.rs rename to virtualization/axvm/src/vm/mod.rs index 41408cf656..c193bdc94b 100644 --- a/virtualization/axvm/src/vm.rs +++ b/virtualization/axvm/src/vm/mod.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Virtual machine state, resources, and lifecycle entry points. + use alloc::{boxed::Box, collections::BTreeMap, format, string::String, sync::Arc, vec::Vec}; use core::{ alloc::Layout, @@ -42,7 +44,7 @@ use crate::{ pub(crate) mod boot; pub(crate) mod memory; -mod prepare; +pub(crate) mod prepare; pub use memory::PreparedMemoryLayout; const VM_ASPACE_BASE: usize = 0x0; @@ -111,7 +113,7 @@ fn write_guest_bytes_to_chunks(chunks: &mut [&mut [u8]], data: &[u8]) -> AxResul pub(crate) struct AxVMResources { // Todo: use more efficient lock. - address_space: AddrSpace, + pub(crate) address_space: AddrSpace, nested_paging: NestedPagingConfig, memory_regions: Vec, config: AxVMConfig, @@ -171,7 +173,10 @@ impl VmRuntimeHandle { Ok(task.cpu_id() as usize) } - #[cfg(target_arch = "loongarch64")] + #[expect( + dead_code, + reason = "only the LoongArch IRQ backend queues physical interrupts" + )] pub(crate) fn queue_external_interrupt( &self, vcpu_id: usize, @@ -258,20 +263,17 @@ impl VmRuntimeHandle { } impl AxVMResources { - fn new(config: AxVMConfig) -> AxResult { - let vcpu_mappings = config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); - let page_table_levels = crate::arch::guest_page_table_levels(&vcpu_mappings)?; - let page_table = crate::arch::new_nested_page_table(page_table_levels)?; + pub(crate) fn from_page_table( + config: AxVMConfig, + page_table: ArchNestedPageTable, + build_nested_paging: impl FnOnce(HostPhysAddr) -> AxResult, + ) -> AxResult { let address_space = AddrSpace::new_empty( page_table, GuestPhysAddr::from(VM_ASPACE_BASE), VM_ASPACE_SIZE, )?; - let nested_paging = crate::arch::nested_paging_config( - address_space.page_table_root(), - page_table_levels, - &vcpu_mappings, - )?; + let nested_paging = build_nested_paging(address_space.page_table_root())?; Ok(Self { address_space, nested_paging, @@ -286,6 +288,10 @@ impl AxVMResources { }) } + pub(crate) const fn config(&self) -> &AxVMConfig { + &self.config + } + fn vcpu_list(&self) -> AxResult<&[AxVCpuRef]> { self.vcpu_list .as_deref() @@ -387,16 +393,22 @@ pub struct AxVM { } impl AxVM { - /// Creates a new VM with the given configuration. - /// Returns an error if the configuration is invalid. - /// The VM is not started until `start` is called. + /// Creates a ready VM with eagerly initialized architecture resources. + /// + /// The VM is not started until [`Self::start`] is called. + /// + /// # Errors + /// + /// Returns an error if nested paging is unsupported for the selected host + /// CPUs or if the initial stage-2 address space cannot be allocated. pub fn new(config: AxVMConfig) -> AxResult { let id = config.id(); let name = config.name(); + let resources = crate::arch::CurrentArch::create_vm_resources(config)?; let result = Arc::new(Self { id, name, - machine: Mutex::new(Machine::Uninit(Box::new(config))), + machine: Mutex::new(Machine::Ready(resources)), pending_fw_cfg: Mutex::new(None), }); @@ -423,33 +435,14 @@ impl AxVM { /// Returns the configured VM interrupt mode. pub fn interrupt_mode(&self) -> VMInterruptMode { - let _ = self.ensure_resources_ready(); self.with_resources(|resources| Ok(resources.config.interrupt_mode())) .unwrap_or(VMInterruptMode::NoIrq) } - fn ensure_resources_ready(&self) -> AxResult { - let mut machine = self.machine.lock(); - if !matches!(machine.status(), VmStatus::Uninit) { - return Ok(()); - } - machine - .prepare_with(|config| { - AxVMResources::new(config).map_err(|err| { - VmLifecycleError::MissingResource(match err { - AxError::NoMemory => "VM address space memory", - _ => "VM address space", - }) - }) - }) - .map_err(VmLifecycleError::into_ax_error) - } - fn with_resources(&self, f: F) -> AxResult where F: FnOnce(&AxVMResources) -> AxResult, { - self.ensure_resources_ready()?; let machine = self.machine.lock(); let resources = machine .resources() @@ -461,7 +454,6 @@ impl AxVM { where F: FnOnce(&mut AxVMResources) -> AxResult, { - self.ensure_resources_ready()?; let mut machine = self.machine.lock(); let resources = machine .resources_mut() @@ -528,20 +520,6 @@ impl AxVM { F: FnOnce(&mut AxVMConfig) -> R, { let mut machine = self.machine.lock(); - if matches!(machine.status(), VmStatus::Uninit) { - let old = core::mem::replace(&mut *machine, Machine::Switching); - if let Machine::Uninit(config) = old { - match AxVMResources::new(*config) { - Ok(resources) => *machine = Machine::Ready(resources), - Err(err) => { - *machine = Machine::Failed(format!("prepare config resources: {err:?}")); - panic!("VM resources are not available: {err:?}"); - } - } - } else { - *machine = old; - } - } let resources = machine .resources_mut() .expect("VM resources are not available for config access"); @@ -585,7 +563,6 @@ impl AxVM { /// Starts the VM by transitioning to Running state. pub fn start(self: &Arc) -> AxResult { - self.ensure_resources_ready()?; if self.status() == VmStatus::Stopped { if let Some(runtime) = self.take_stopped_runtime() { runtime.join_all_vcpu_tasks(self.id()); @@ -841,7 +818,6 @@ impl AxVM { Ok(()) } - #[cfg(not(target_arch = "aarch64"))] pub(crate) fn handle_nested_page_fault( &self, addr: GuestPhysAddr, @@ -857,7 +833,6 @@ impl AxVM { .unwrap_or(false) } - #[cfg(not(target_arch = "aarch64"))] fn debug_nested_page_fault( vm_id: usize, resources: &AxVMResources, @@ -1247,7 +1222,7 @@ impl AxVM { VmStatus::Running | VmStatus::Paused | VmStatus::Stopping => { self.stop_and_join_runtime(StopReason::Forced)?; } - VmStatus::Ready | VmStatus::Stopped | VmStatus::Uninit | VmStatus::Failed => { + VmStatus::Ready | VmStatus::Stopped | VmStatus::Failed => { if let Some(runtime) = self.take_stopped_runtime() { runtime.join_all_vcpu_tasks(vm_id); } diff --git a/virtualization/axvm/src/vm/prepare.rs b/virtualization/axvm/src/vm/prepare.rs index c609e1befd..4f57cd84ba 100644 --- a/virtualization/axvm/src/vm/prepare.rs +++ b/virtualization/axvm/src/vm/prepare.rs @@ -1,8 +1,8 @@ -//! VM preparation orchestration. +//! Architecture-neutral mechanics used by architecture-owned VM initialization. -mod address_space; -mod devices; -mod vcpus; +pub(crate) mod address_space; +pub(crate) mod devices; +pub(crate) mod vcpus; use alloc::{format, sync::Arc}; @@ -10,35 +10,32 @@ use ax_errno::{AxResult, ax_err, ax_err_type}; use axdevice::{DeviceFactoryRegistry, register_builtin_factories}; use self::{devices::PreparedDevices, vcpus::PreparedVcpus}; -use super::AxVM; +use super::{AxVM, AxVMResources}; use crate::irq::InterruptFabric; +pub(crate) enum VmInitRequest<'a> { + Default, + Provided { + factories: &'a DeviceFactoryRegistry, + interrupt_fabric: InterruptFabric, + }, +} + +pub(crate) struct PreparedVm { + vcpus: PreparedVcpus, + devices: PreparedDevices, +} + +impl PreparedVm { + pub(crate) const fn new(vcpus: PreparedVcpus, devices: PreparedDevices) -> Self { + Self { vcpus, devices } + } +} + impl AxVM { /// Sets up the VM before booting. pub fn prepare(&self) -> AxResult { - self.ensure_resources_ready()?; - let mut factories = DeviceFactoryRegistry::new(); - register_builtin_factories(&mut factories)?; - let interrupt_mode = self.interrupt_mode(); - #[cfg(target_arch = "riscv64")] - let interrupt_fabric = { - let machine = self.machine.lock(); - let resources = machine.resources().ok_or_else(|| { - ax_err_type!( - BadState, - "VM resources are not available for RISC-V IRQ setup" - ) - })?; - crate::irq::riscv::configure( - &mut factories, - interrupt_mode, - resources.config.emu_devices(), - )? - }; - #[cfg(not(target_arch = "riscv64"))] - let interrupt_fabric = InterruptFabric::new(interrupt_mode); - - self.prepare_with_factories(&factories, interrupt_fabric) + crate::arch::CurrentArch::init_vm(self, VmInitRequest::Default) } /// Sets up the VM with explicit device factories and an interrupt fabric. @@ -47,52 +44,73 @@ impl AxVM { factories: &DeviceFactoryRegistry, interrupt_fabric: InterruptFabric, ) -> AxResult { - self.ensure_resources_ready()?; - let mut machine = self.machine.lock(); - if !matches!( - machine.status(), - crate::lifecycle::VmStatus::Ready | crate::lifecycle::VmStatus::Stopped - ) { - return ax_err!( - BadState, - format!( - "VM[{}] cannot prepare from {:?}", - self.id(), - machine.status() - ) - ); - } - let resources = machine - .resources_mut() - .ok_or_else(|| ax_err_type!(BadState, "VM resources are not available for prepare"))?; - if resources.vcpu_list.is_some() - || resources.devices.is_some() - || resources.interrupt_fabric.is_some() - { - resources.reset_transient_resources()?; - } - interrupt_fabric.validate_mode(resources.config.interrupt_mode())?; + crate::arch::CurrentArch::init_vm( + self, + VmInitRequest::Provided { + factories, + interrupt_fabric, + }, + ) + } +} - let dtb_addr = resources.config.image_config().dtb_load_gpa; - let vcpus = PreparedVcpus::create(self.id(), resources, dtb_addr)?; - let devices = PreparedDevices::build(self, resources, factories, &interrupt_fabric)?; +pub(crate) fn default_device_factories() -> AxResult { + let mut factories = DeviceFactoryRegistry::new(); + register_builtin_factories(&mut factories)?; + Ok(factories) +} - if dtb_addr.is_some() && resources.boot_description.device_tree().is_none() { - return ax_err!( - InvalidInput, - "DTB load GPA is configured but no guest device tree bytes are registered" - ); - } +pub(crate) fn complete_vm_init( + vm: &AxVM, + interrupt_fabric: InterruptFabric, + initialize: impl FnOnce(&mut AxVMResources, &InterruptFabric) -> AxResult, +) -> AxResult { + let mut machine = vm.machine.lock(); + if !matches!( + machine.status(), + crate::lifecycle::VmStatus::Ready | crate::lifecycle::VmStatus::Stopped + ) { + return ax_err!( + BadState, + format!("VM[{}] cannot prepare from {:?}", vm.id(), machine.status()) + ); + } + let resources = machine + .resources_mut() + .ok_or_else(|| ax_err_type!(BadState, "VM resources are not available for prepare"))?; + resources.reset_transient_resources()?; + interrupt_fabric.validate_mode(resources.config.interrupt_mode())?; - address_space::map_guest_address_space(self, resources, devices.devices())?; - vcpus.setup(resources)?; + let prepared = match initialize(resources, &interrupt_fabric) { + Ok(prepared) => prepared, + Err(err) => { + if let Err(reset_err) = resources.reset_transient_resources() { + warn!( + "VM[{}] failed to reset transient resources after initialization error: \ + {reset_err:?}", + vm.id() + ); + } + return Err(err); + } + }; + resources.phys_cpu_ls = resources.config.phys_cpu_ls.clone(); + resources.vcpu_list = Some(prepared.vcpus.into_boxed_slice()); + resources.devices = Some(Arc::new(prepared.devices.into_inner())); + resources.interrupt_fabric = Some(interrupt_fabric); - resources.phys_cpu_ls = resources.config.phys_cpu_ls.clone(); - resources.vcpu_list = Some(vcpus.into_boxed_slice()); - resources.devices = Some(Arc::new(devices.into_inner())); - resources.interrupt_fabric = Some(interrupt_fabric); + info!("VM setup: id={}", vm.id()); + Ok(()) +} - info!("VM setup: id={}", self.id()); - Ok(()) +pub(crate) fn validate_guest_dtb(resources: &AxVMResources) -> AxResult { + if resources.config.image_config().dtb_load_gpa.is_some() + && resources.boot_description.device_tree().is_none() + { + return ax_err!( + InvalidInput, + "DTB load GPA is configured but no guest device tree bytes are registered" + ); } + Ok(()) } diff --git a/virtualization/axvm/src/vm/prepare/address_space.rs b/virtualization/axvm/src/vm/prepare/address_space.rs index 9c8d76ca6e..c829feeb37 100644 --- a/virtualization/axvm/src/vm/prepare/address_space.rs +++ b/virtualization/axvm/src/vm/prepare/address_space.rs @@ -5,20 +5,16 @@ use alloc::vec::Vec; use ax_errno::AxResult; use axdevice::AxVmDevices; use axdevice_base::Resource; -#[cfg(all(target_arch = "x86_64", feature = "vmx"))] -use axvm_types::GuestPhysAddr; -#[cfg(all(target_arch = "x86_64", feature = "vmx"))] -use x86_vcpu::X86_APIC_ACCESS_GPA; use super::super::{AxVM, AxVMResources, VM_ASPACE_BASE, VM_ASPACE_SIZE}; use crate::layout::{GuestOwnedRegion, VmRegionKind, build_address_layout}; -pub(super) fn map_guest_address_space( +pub(crate) fn map_guest_address_space( vm: &AxVM, resources: &mut AxVMResources, devices: &AxVmDevices, + owned_regions: &[GuestOwnedRegion], ) -> AxResult { - let owned_regions = guest_owned_regions(resources); let emulated_resources = devices .devices() .flat_map(|device| device.resources().iter().cloned()) @@ -29,7 +25,7 @@ pub(super) fn map_guest_address_space( VM_ASPACE_SIZE, resources.config.pass_through_devices(), resources.config.pass_through_addresses(), - &owned_regions, + owned_regions, &emulated_resources, )?; @@ -53,20 +49,10 @@ pub(super) fn map_guest_address_space( } resources.address_layout = Some(address_layout); - #[cfg(all(target_arch = "x86_64", feature = "vmx"))] - resources.address_space.map_linear( - GuestPhysAddr::from(X86_APIC_ACCESS_GPA), - crate::arch::x86_apic_access_page_addr(), - ax_memory_addr::PAGE_SIZE_4K, - axvm_types::MappingFlags::DEVICE - | axvm_types::MappingFlags::READ - | axvm_types::MappingFlags::WRITE, - )?; - Ok(()) } -fn guest_owned_regions(resources: &AxVMResources) -> Vec { +pub(crate) fn guest_owned_regions(resources: &AxVMResources) -> Vec { let mut regions = resources .memory_regions .iter() @@ -93,12 +79,5 @@ fn guest_owned_regions(resources: &AxVMResources) -> Vec { }), ); - #[cfg(all(target_arch = "x86_64", feature = "vmx"))] - regions.push(GuestOwnedRegion::new( - X86_APIC_ACCESS_GPA, - ax_memory_addr::PAGE_SIZE_4K, - VmRegionKind::Reserved, - )); - regions } diff --git a/virtualization/axvm/src/vm/prepare/devices.rs b/virtualization/axvm/src/vm/prepare/devices.rs index 23700f5bc8..5f24eee57a 100644 --- a/virtualization/axvm/src/vm/prepare/devices.rs +++ b/virtualization/axvm/src/vm/prepare/devices.rs @@ -1,35 +1,23 @@ //! Device construction for VM preparation. -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -use alloc::{format, sync::Arc}; - use ax_errno::AxResult; -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -use ax_errno::ax_err_type; use axdevice::{AxVmDeviceConfig, AxVmDevices, DeviceBuildContext, DeviceFactoryRegistry}; -#[cfg(target_arch = "aarch64")] -use axdevice_base::DeviceRegistry as _; -#[cfg(target_arch = "x86_64")] -use axdevice_base::{BaseDeviceOps, DeviceRegistry as _, PortDeviceAdapter}; use super::super::{AxVM, AxVMResources}; -#[cfg(target_arch = "aarch64")] -use crate::config::VMInterruptMode; use crate::irq::InterruptFabric; -pub(super) struct PreparedDevices { - devices: AxVmDevices, +pub(crate) struct PreparedDevices { + pub(crate) devices: AxVmDevices, } impl PreparedDevices { - pub(super) fn build( - vm: &AxVM, + pub(crate) fn build_common( resources: &AxVMResources, factories: &DeviceFactoryRegistry, interrupt_fabric: &InterruptFabric, ) -> AxResult { let build_context = DeviceBuildContext::new(interrupt_fabric); - let mut devices = AxVmDevices::build_with_factories( + let devices = AxVmDevices::build_with_factories( AxVmDeviceConfig { emu_configs: resources.config.emu_devices().to_vec(), }, @@ -37,92 +25,18 @@ impl PreparedDevices { &build_context, )?; - #[cfg(target_arch = "x86_64")] - for port in resources.config.pass_through_ports() { - let passthrough = Arc::new(crate::host::x86_port::HostPortPassthrough::new( - port.base, - port.length, - )?); - let range = passthrough.address_range(); - debug!( - "PT port region: [{:#x}~{:#x}]", - range.start.number(), - range.end.number(), - ); - devices - .register(PortDeviceAdapter::from_arc(passthrough)) - .map_err(|err| ax_err_type!(InvalidInput, format!("register PT port: {err:?}")))?; - } - - Self::register_arch_devices(vm, resources, &mut devices)?; - vm.add_special_emulated_devices(&mut devices)?; Ok(Self { devices }) } - pub(super) const fn devices(&self) -> &AxVmDevices { - &self.devices - } - - pub(super) fn into_inner(self) -> AxVmDevices { - self.devices - } - - #[cfg(target_arch = "aarch64")] - fn register_arch_devices( - vm: &AxVM, - resources: &AxVMResources, - devices: &mut AxVmDevices, - ) -> AxResult { - let passthrough = resources.config.interrupt_mode() == VMInterruptMode::Passthrough; - if passthrough { - let spis = resources.config.pass_through_spis(); - let cpu_id = vm.id() - 1; // FIXME: get the real CPU id. - let mut gicd_found = false; - - for device in devices.devices() { - if let Some(gicd) = device.as_any().downcast_ref::() { - debug!("VGicD found, assigning SPIs..."); - - for spi in spis { - gicd.assign_irq(*spi + 32, cpu_id, (0, 0, 0, cpu_id as _)) - } - - gicd_found = true; - break; - } - } - - if !gicd_found { - warn!("Failed to assign SPIs: No VGicD found in device list"); - } - } else { - for dev in axdevice::create_vtimer_devices() { - devices - .register(Arc::from(dev) as Arc) - .map_err(|e| ax_err_type!(InvalidInput, format!("register vtimer: {e:?}")))?; - } - } - Ok(()) + pub(crate) fn register_special_devices(&mut self, vm: &AxVM) -> AxResult { + vm.add_special_emulated_devices(&mut self.devices) } - #[cfg(target_arch = "x86_64")] - fn register_arch_devices( - _vm: &AxVM, - resources: &AxVMResources, - devices: &mut AxVmDevices, - ) -> AxResult { - for config in resources.config.emu_devices() { - crate::arch::register_x86_arch_device(config, devices)?; - } - Ok(()) + pub(crate) const fn devices(&self) -> &AxVmDevices { + &self.devices } - #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] - fn register_arch_devices( - _vm: &AxVM, - _resources: &AxVMResources, - _devices: &mut AxVmDevices, - ) -> AxResult { - Ok(()) + pub(crate) fn into_inner(self) -> AxVmDevices { + self.devices } } diff --git a/virtualization/axvm/src/vm/prepare/vcpus.rs b/virtualization/axvm/src/vm/prepare/vcpus.rs index d3081e4641..2012650198 100644 --- a/virtualization/axvm/src/vm/prepare/vcpus.rs +++ b/virtualization/axvm/src/vm/prepare/vcpus.rs @@ -1,44 +1,41 @@ -//! vCPU construction and setup for VM preparation. +//! Architecture-neutral vCPU collection construction and setup. use alloc::{boxed::Box, sync::Arc, vec::Vec}; use ax_errno::AxResult; -use axvm_types::{EmulatedDeviceType, GuestPhysAddr}; +use axvm_types::VmArchVcpuOps; use super::super::{AxVCpuRef, AxVMResources, VCpu}; -use crate::{ - arch::{ArchOps, CurrentArch, VcpuCreateContext, VcpuSetupContext}, - config::{GuestBootPolicy, VMBootProtocol}, -}; -pub(super) struct PreparedVcpus { +#[derive(Clone, Copy, Debug)] +pub(crate) struct VcpuPlacement { + pub(crate) id: usize, + pub(crate) phys_cpu_set: Option, + pub(crate) phys_cpu_id: usize, +} + +pub(crate) struct PreparedVcpus { vcpus: Vec, } impl PreparedVcpus { - pub(super) fn create( + pub(crate) fn create( vm_id: usize, - resources: &AxVMResources, - dtb_addr: Option, + placements: &[VcpuPlacement], + mut build_config: impl FnMut( + VcpuPlacement, + ) + -> AxResult<::CreateConfig>, ) -> AxResult { - let vcpu_id_pcpu_sets = resources.config.phys_cpu_ls.get_vcpu_affinities_pcpu_ids(); - let create_state = CurrentArch::new_vcpu_create_state(&vcpu_id_pcpu_sets)?; - let firmware_boot = guest_uses_firmware_boot(resources); - - debug!("dtb_load_gpa: {dtb_addr:?}"); - debug!("id: {vm_id}, VCpuIdPCpuSets: {vcpu_id_pcpu_sets:#x?}"); + debug!("id: {vm_id}, vCPU placements: {placements:#x?}"); - let mut vcpus = Vec::with_capacity(vcpu_id_pcpu_sets.len()); - for (vcpu_id, phys_cpu_set, phys_cpu_id) in vcpu_id_pcpu_sets { - let arch_config = CurrentArch::build_vcpu_create_config( - &create_state, - VcpuCreateContext { - vcpu_id, - phys_cpu_id, - dtb_addr, - firmware_boot, - }, - )?; + let mut vcpus = Vec::with_capacity(placements.len()); + for placement in placements.iter().copied() { + trace!( + "Creating VM[{vm_id}] vCPU[{}] for physical CPU {}", + placement.id, placement.phys_cpu_id + ); + let arch_config = build_config(placement)?; // FIXME: VCpu is neither `Send` nor `Sync` by design, check whether // 1. we should make it `Send` and `Sync`, or @@ -46,8 +43,8 @@ impl PreparedVcpus { #[allow(clippy::arc_with_non_send_sync)] vcpus.push(Arc::new(VCpu::new( vm_id, - vcpu_id, - phys_cpu_set, + placement.id, + placement.phys_cpu_set, arch_config, )?)); } @@ -55,20 +52,17 @@ impl PreparedVcpus { Ok(Self { vcpus }) } - pub(super) fn setup(&self, resources: &AxVMResources) -> AxResult { + pub(crate) fn setup( + &self, + resources: &AxVMResources, + mut build_config: impl FnMut( + &crate::config::AxVMConfig, + &[crate::vm::VMMemoryRegion], + ) + -> AxResult<::SetupConfig>, + ) -> AxResult { for vcpu in &self.vcpus { - let setup_config = CurrentArch::build_vcpu_setup_config(VcpuSetupContext { - interrupt_mode: resources.config.interrupt_mode(), - emulates_console: resources - .config - .emu_devices() - .iter() - .any(|dev| dev.emu_type == EmulatedDeviceType::Console), - passthrough_ports: resources.config.pass_through_ports(), - memory_regions: &resources.memory_regions, - firmware_boot: guest_uses_firmware_boot(resources), - })?; - + let setup_config = build_config(&resources.config, &resources.memory_regions)?; let entry = if vcpu.id() == 0 { resources.config.bsp_entry() } else { @@ -76,22 +70,26 @@ impl PreparedVcpus { }; debug!("Setting up vCPU[{}] entry at {:#x}", vcpu.id(), entry); - vcpu.setup(entry, resources.nested_paging, setup_config)?; } Ok(()) } - pub(super) fn into_boxed_slice(self) -> Box<[AxVCpuRef]> { + pub(crate) fn into_boxed_slice(self) -> Box<[AxVCpuRef]> { self.vcpus.into_boxed_slice() } } -fn guest_uses_firmware_boot(resources: &AxVMResources) -> bool { - matches!( - resources.config.boot_policy(), - GuestBootPolicy::AdjustKernelForBootProtocol { - protocol: VMBootProtocol::Uefi, - } - ) +pub(crate) fn vcpu_placements(resources: &AxVMResources) -> Vec { + resources + .config + .phys_cpu_ls + .get_vcpu_affinities_pcpu_ids() + .into_iter() + .map(|(id, phys_cpu_set, phys_cpu_id)| VcpuPlacement { + id, + phys_cpu_set, + phys_cpu_id, + }) + .collect() } diff --git a/virtualization/axvm/tests/arch_boundary_contract.rs b/virtualization/axvm/tests/arch_boundary_contract.rs index d7861d0d9f..a7c76a4eb5 100644 --- a/virtualization/axvm/tests/arch_boundary_contract.rs +++ b/virtualization/axvm/tests/arch_boundary_contract.rs @@ -14,10 +14,9 @@ #[test] fn vm_core_does_not_handle_arch_local_exits() { - let vm_rs = include_str!("../src/vm.rs"); + let vm_rs = include_str!("../src/vm/mod.rs"); for forbidden in [ - "CurrentArch", "ArchOps", "CurrentArch::handle_vcpu_exit", "VcpuRunAction", @@ -25,11 +24,97 @@ fn vm_core_does_not_handle_arch_local_exits() { ] { assert!( !vm_rs.contains(forbidden), - "vm.rs must not contain architecture-local exit handling detail: {forbidden}" + "vm/mod.rs must not contain architecture-local exit handling detail: {forbidden}" ); } } +#[test] +fn common_vm_initialization_only_uses_high_level_arch_entrypoints() { + let vm = include_str!("../src/vm/mod.rs"); + let preparation = include_str!("../src/vm/prepare.rs"); + let common_sources = [ + vm, + preparation, + include_str!("../src/vm/prepare/address_space.rs"), + include_str!("../src/vm/prepare/devices.rs"), + include_str!("../src/vm/prepare/vcpus.rs"), + ]; + + assert!(vm.contains("CurrentArch::create_vm_resources(config)")); + assert!(preparation.contains("CurrentArch::init_vm")); + + for source in &common_sources { + for line in source.lines().filter(|line| line.contains("CurrentArch::")) { + assert!( + line.contains("CurrentArch::create_vm_resources") + || line.contains("CurrentArch::init_vm"), + "common VM initialization calls a fine-grained architecture hook: {line}" + ); + } + } + + for forbidden in [ + "configure_interrupt_fabric", + "register_arch_devices", + "append_arch_owned_regions", + "map_arch_address_space", + "new_vcpu_create_state", + "build_vcpu_create_config", + "build_vcpu_setup_config", + ] { + assert!( + common_sources + .iter() + .all(|source| !source.contains(forbidden)), + "common VM initialization must not call architecture step hook: {forbidden}" + ); + } +} + +#[test] +fn every_architecture_owns_vm_resource_creation_and_initialization() { + for source in [ + include_str!("../src/arch/aarch64/vm.rs"), + include_str!("../src/arch/loongarch64/vm.rs"), + include_str!("../src/arch/riscv64/vm.rs"), + include_str!("../src/arch/x86_64/vm.rs"), + ] { + assert!(source.contains("fn create_vm_resources")); + assert!(source.contains("fn init_vm")); + } +} + +#[test] +fn custom_vm_init_inputs_cross_the_arch_boundary_unchanged() { + let preparation = include_str!("../src/vm/prepare.rs"); + assert!(preparation.contains("VmInitRequest::Provided")); + assert!(preparation.contains("factories,")); + assert!(preparation.contains("interrupt_fabric,")); + + for source in [ + include_str!("../src/arch/aarch64/vm.rs"), + include_str!("../src/arch/loongarch64/vm.rs"), + include_str!("../src/arch/riscv64/vm.rs"), + include_str!("../src/arch/x86_64/vm.rs"), + ] { + assert!(source.contains("VmInitRequest::Provided")); + assert!(source.contains("init_vm_with(vm, factories, interrupt_fabric)")); + } +} + +#[test] +fn failed_vm_initialization_resets_transient_resources_before_retry() { + let preparation = include_str!("../src/vm/prepare.rs"); + let initialize = preparation + .split_once("let prepared = match initialize") + .expect("VM initialization must handle architecture errors") + .1; + + assert!(initialize.contains("resources.reset_transient_resources()")); + assert!(initialize.contains("return Err(err)")); +} + #[test] fn runtime_vcpu_loop_only_consumes_scheduler_actions() { let runtime_vcpus_rs = include_str!("../src/runtime/vcpus.rs"); @@ -45,3 +130,264 @@ fn runtime_vcpu_loop_only_consumes_scheduler_actions() { ); } } + +#[test] +fn production_sources_keep_architecture_cfg_inside_arch_module() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let violations = find_target_arch_cfg_outside_arch(&source_root, &source_root); + + assert!( + violations.is_empty(), + "target_arch must stay inside src/arch; found: {}", + violations.join(", ") + ); +} + +#[test] +fn arch_root_contains_only_architecture_directories_and_dispatch_page() { + let arch_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/arch"); + let mut unexpected_entries = std::fs::read_dir(&arch_root) + .expect("AxVM architecture directory must be readable") + .map(|entry| entry.expect("AxVM architecture entry must be readable")) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| { + !matches!( + name.as_str(), + "aarch64" | "loongarch64" | "riscv64" | "x86_64" | "mod.rs" + ) + }) + .collect::>(); + unexpected_entries.sort(); + + assert!( + unexpected_entries.is_empty(), + "arch root must contain only architecture directories and the dispatch page; found: {}", + unexpected_entries.join(", ") + ); +} + +#[test] +fn arch_dispatch_page_does_not_own_common_implementations() { + let dispatch = include_str!("../src/arch/mod.rs"); + + for forbidden in [ + "#[path", + "trait ArchOps", + "struct MmioReadExit", + "fn handle_mmio_read", + "fn default_vcpu_affinities", + ] { + assert!( + !dispatch.contains(forbidden), + "arch/mod.rs must only select and export the current architecture: {forbidden}" + ); + } +} + +#[test] +fn common_domains_live_outside_architecture_directories() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + + for relative_path in [ + "boot/fdt/mod.rs", + "boot/images/mod.rs", + "host/arceos.rs", + "npt.rs", + ] { + assert!( + source_root.join(relative_path).is_file(), + "common AxVM domain must use its canonical source path: {relative_path}" + ); + } +} + +#[test] +fn vm_domain_uses_a_canonical_directory_module() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + + assert!( + source_root.join("vm/mod.rs").is_file(), + "the VM domain with child modules must use vm/mod.rs as its directory page" + ); + assert!( + !source_root.join("vm.rs").exists(), + "vm.rs must not coexist with the vm child-module directory" + ); +} + +#[test] +fn common_modules_do_not_include_architecture_sources() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut violations = Vec::new(); + find_source_files(&source_root, &mut |path, source| { + if !path.starts_with(source_root.join("arch")) + && source.contains("#[path") + && source.contains("arch/") + { + violations.push(source_relative_path(&source_root, path)); + } + }); + + assert!( + violations.is_empty(), + "common modules must not include implementations from src/arch: {}", + violations.join(", ") + ); +} + +#[test] +fn architecture_directories_only_select_their_own_target() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let arch_root = source_root.join("arch"); + let architectures = ["aarch64", "loongarch64", "riscv64", "x86_64"]; + let mut violations = Vec::new(); + + for architecture in architectures { + find_source_files(&arch_root.join(architecture), &mut |path, source| { + for other_architecture in architectures { + if other_architecture != architecture + && source.contains(&format!("target_arch = \"{other_architecture}\"")) + { + violations.push(format!( + "{} selects {other_architecture}", + source_relative_path(&source_root, path) + )); + } + } + }); + } + + assert!( + violations.is_empty(), + "an architecture directory must not select another target: {}", + violations.join(", ") + ); +} + +#[test] +fn axvisor_vm_creation_uses_unified_guest_boot_facade() { + let axvisor_config = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../os/axvisor/src/config.rs"); + let source = std::fs::read_to_string(&axvisor_config) + .expect("Axvisor VM creation source must be readable"); + + for legacy_call in [ + "handle_fdt_operations", + "ImageLoader::new", + "x86_linux_direct_boot_config", + "DEFAULT_X86_BIOS_LOAD_GPA", + ] { + assert!( + !source.contains(legacy_call), + "Axvisor VM creation must use the unified AxVM boot facade: {legacy_call}" + ); + } +} + +#[test] +fn host_time_trait_only_exposes_common_clock_capabilities() { + let host_traits = include_str!("../src/host/traits.rs"); + + for architecture_specific_detail in ["CancelToken", "fn register_timer"] { + assert!( + !host_traits.contains(architecture_specific_detail), + "HostTime must not expose architecture-specific timer details: \ + {architecture_specific_detail}" + ); + } +} + +#[test] +fn vcpu_setup_context_keeps_named_capabilities() { + let types = include_str!("../src/architecture/types.rs"); + let ops = include_str!("../src/architecture/ops.rs"); + let preparation = include_str!("../src/vm/prepare/vcpus.rs"); + + assert!( + [&types, &ops, &preparation] + .into_iter() + .all(|source| !source.contains("VcpuSetupContext")), + "vCPU setup must pass named configuration and memory sources without a union context" + ); +} + +#[test] +fn vm_init_capability_traits_are_not_reintroduced() { + let capabilities = include_str!("../src/architecture/capabilities.rs"); + let ops = include_str!("../src/architecture/ops.rs"); + + for forbidden in [ + "trait DevicePlatform", + "trait AddressSpacePlatform", + "VcpuCreateContext", + "fn build_vcpu_create_config", + "fn build_vcpu_setup_config", + ] { + assert!( + !capabilities.contains(forbidden) && !ops.contains(forbidden), + "VM initialization detail must stay behind CurrentArch::init_vm: {forbidden}" + ); + } +} + +#[test] +fn eager_vm_lifecycle_has_no_uninit_state() { + let status = include_str!("../src/lifecycle/status.rs"); + let machine = include_str!("../src/lifecycle/machine.rs"); + let vm = include_str!("../src/vm/mod.rs"); + + assert!(!status.contains("Uninit")); + assert!(!machine.contains("Machine::Uninit")); + assert!(vm.contains("machine: Mutex::new(Machine::Ready(resources))")); +} + +fn find_target_arch_cfg_outside_arch( + source_root: &std::path::Path, + directory: &std::path::Path, +) -> Vec { + let mut violations = Vec::new(); + for entry in std::fs::read_dir(directory).expect("AxVM source directory must be readable") { + let entry = entry.expect("AxVM source directory entry must be readable"); + let path = entry.path(); + if path.is_dir() { + if path != source_root.join("arch") { + violations.extend(find_target_arch_cfg_outside_arch(source_root, &path)); + } + continue; + } + + if path.extension().is_some_and(|extension| extension == "rs") + && std::fs::read_to_string(&path) + .expect("AxVM source file must be readable") + .contains("target_arch") + { + violations.push( + path.strip_prefix(source_root) + .expect("source path must be below src") + .display() + .to_string(), + ); + } + } + violations +} + +fn find_source_files(directory: &std::path::Path, visit: &mut impl FnMut(&std::path::Path, &str)) { + for entry in std::fs::read_dir(directory).expect("AxVM source directory must be readable") { + let entry = entry.expect("AxVM source directory entry must be readable"); + let path = entry.path(); + if path.is_dir() { + find_source_files(&path, visit); + } else if path.extension().is_some_and(|extension| extension == "rs") { + let source = std::fs::read_to_string(&path).expect("AxVM source file must be readable"); + visit(&path, &source); + } + } +} + +fn source_relative_path(source_root: &std::path::Path, path: &std::path::Path) -> String { + path.strip_prefix(source_root) + .expect("source path must be below src") + .display() + .to_string() +}