diff --git a/Cargo.lock b/Cargo.lock index 1cbe9b7e0b..2996293d1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4608,13 +4608,14 @@ checksum = "753ac4e7753fa007b8f7828d0e01e9ececa48b689460e6380fb36bbf74654a8d" [[package]] name = "kbpf-basic" -version = "0.5.7" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c066bb7997a6daf34a35ad2790570f32f33677dec17538781bbaff4be2248e" +checksum = "939c71c68bfebd52de856c50f7650713e449fe8a83255e8cee53d8519500b5e0" dependencies = [ "ax-errno 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 2.13.0", "int-enum", + "lock_api", "log", "lru 0.18.0", "printf-compat", @@ -4690,9 +4691,9 @@ dependencies = [ [[package]] name = "kprobe" -version = "0.5.5" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cf6b7b70ff393b62d067f6f2e35bd64c9278ac77fd6fa2de883b2c0be8fa5ff" +checksum = "58aa2f033d7576265cbdd2e98b902e9f10d4c3d7dc0e95f897dd59f2af9e480a" dependencies = [ "cfg-if", "lock_api", diff --git a/os/StarryOS/docs/loongarch64-dmw-kernel-aspace.md b/os/StarryOS/docs/loongarch64-dmw-kernel-aspace.md new file mode 100644 index 0000000000..45d925900e --- /dev/null +++ b/os/StarryOS/docs/loongarch64-dmw-kernel-aspace.md @@ -0,0 +1,43 @@ +# LoongArch64 DMW and Kernel Address Space + +LoongArch64 platforms may enable DMW (Direct Mapping Window) for the cached +physical direct map. In the QEMU virt platform, `phys_to_virt()` uses the +`0x9000_0000_0000_0000` DMW window: + +```text +VA = 0x9000_0000_0000_0000 + PA +``` + +This range is translated by DMW hardware and does not consult the kernel page +table. Therefore it must not be treated as ordinary page-table-backed kernel +virtual memory. + +The page-table-backed kernel address space should use a non-DMW address range, +and must still be a legal mapped virtual address. With the current +LoongArch64 page-table configuration (`VALEN = 48`), bits `[63:48]` must be a +sign extension of bit 47. For example: + +```toml +kernel-aspace-base = "0xFFFF_8000_0000_0000" +kernel-aspace-size = "0x0000_7fff_ffff_f000" +``` + +Temporary kernel mappings such as `vmap`, eBPF ring-buffer aliases, MMIO +`iomap` ranges, module memory, and trampoline pages should be allocated from +this page-table-backed kernel address space. + +Do not add a second page-table mapping for DMW direct-map RAM. Besides being +redundant, it can conflict with real kernel virtual mappings because the current +LoongArch64 page-table implementation indexes only the low 48 virtual-address +bits. For example, mappings in `0x9000_...` and page-table-backed mappings can +share the same page-table indexes if their low 48 bits are equal. + +`ax-mm` therefore maps a physical memory region through `phys_to_virt()` only +when the resulting virtual range is contained in the configured kernel address +space. On DMW platforms this skips the hardware direct-map range; on platforms +where the direct map is page-table-backed, the existing mapping behavior is +preserved. + + +## References +https://www.kernel.org/doc/html/v6.1/loongarch/introduction.html#virtual-memory \ No newline at end of file diff --git a/os/StarryOS/kernel/Cargo.toml b/os/StarryOS/kernel/Cargo.toml index 286f4df557..164b2526c9 100644 --- a/os/StarryOS/kernel/Cargo.toml +++ b/os/StarryOS/kernel/Cargo.toml @@ -16,6 +16,7 @@ license.workspace = true [features] default = ["dynamic_debug"] dev-log = [] +kprobe_test = [] ext4 = ["ax-fs/ext4"] input = ["dep:ax-input", "ax-feat/input"] memtrack = ["ax-feat/backtrace", "ax-alloc/tracking"] @@ -134,7 +135,7 @@ zerocopy = { version = "0.8", features = ["derive"] } ax-ipi = { workspace = true } static-keys = "0.8" ddebug = "0.5" -kprobe = "0.5" +kprobe = "0.6" kmod = { version = "0.2", package = "kmod-tools" } kmod-loader = "0.2.1" lwprintf-rs = "0.3" @@ -145,7 +146,7 @@ sg200x-bsp = { workspace = true, optional = true } tock-registers = { version = "0.9", optional = true } ktracepoint = "0.6" ksym = "0.6" -kbpf-basic = "0.5.7" +kbpf-basic = "0.6" rbpf = { version = "0.4", default-features = false } [target.'cfg(target_arch = "x86_64")'.dependencies] diff --git a/os/StarryOS/kernel/src/ebpf/map.rs b/os/StarryOS/kernel/src/ebpf/map.rs index bd1c0a597c..3bd974fc00 100644 --- a/os/StarryOS/kernel/src/ebpf/map.rs +++ b/os/StarryOS/kernel/src/ebpf/map.rs @@ -3,11 +3,11 @@ //! to tgoskits' `ax_hal` / `ax_kspin` / `ax_errno` / `ax_alloc` package //! names per `crate-fork-audit.md §6`. -use alloc::{borrow::Cow, sync::Arc}; +use alloc::{borrow::Cow, sync::Arc, vec::Vec}; use core::ops::{Deref, DerefMut}; use ax_errno::{AxError, AxResult}; -use ax_kspin::{SpinNoPreempt, SpinNoPreemptGuard}; +use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr}; use axpoll::{PollSet, Pollable}; use kbpf_basic::{ PollWaker, @@ -17,13 +17,15 @@ use kbpf_basic::{ use crate::{ ebpf::transform::{EbpfKernelAuxiliary, PerCpuImpl}, file::{FileLike, Kstat}, + kprobe::KernelRawMutex, + pseudofs::DeviceMmap, }; /// File-like handle for a BPF map. Holds the `UnifiedMap` (the kbpf-basic /// abstraction over array / hash / lru / queue / perf-array maps) and a /// `PollSet` so `poll(2)`-based maps (e.g. ringbuf) can wake waiters. pub struct BpfMap { - unified_map: SpinNoPreempt, + unified_map: Arc>, poll_ready: Arc, } @@ -35,16 +37,16 @@ impl core::fmt::Debug for BpfMap { impl BpfMap { /// Wrap a freshly-created `UnifiedMap` in the kernel file-like layer. - pub fn new(unified_map: UnifiedMap, poll_ready: Arc) -> Self { + pub fn new(unified_map: UnifiedMap, poll_ready: Arc) -> Self { BpfMap { - unified_map: SpinNoPreempt::new(unified_map), + unified_map: Arc::new(unified_map), poll_ready, } } /// Lock and access the underlying `UnifiedMap`. - pub fn unified_map(&self) -> SpinNoPreemptGuard<'_, UnifiedMap> { - self.unified_map.lock() + pub fn unified_map(&self) -> &UnifiedMap { + self.unified_map.as_ref() } } @@ -82,6 +84,28 @@ impl FileLike for BpfMap { fn path(&self) -> Cow<'_, str> { "anon_inode:[bpf_map]".into() } + + fn device_mmap(&self, offset: u64, length: u64) -> AxResult { + // for ringbuf maps, userland calls mmap on the map fd to get a pointer to the ringbuf; + // the kernel must support this. For other map types, mmap is not meaningful and Linux rejects it with EINVAL. + if !offset.is_multiple_of(PAGE_SIZE_4K as u64) + || !length.is_multiple_of(PAGE_SIZE_4K as u64) + { + return Err(AxError::InvalidInput); + } + + let unified_map = self.unified_map(); + let map = unified_map.map(); + let phy_addrs = map + .map_mmap(offset as usize, length as usize) + .map_err(|_| AxError::InvalidInput)? + .iter() + .map(|&phys_addr| PhysAddr::from_usize(phys_addr)) + .collect::>(); + + let retain: Arc = self.unified_map.clone(); + Ok(DeviceMmap::PhysicalPages(phy_addrs, Some(retain))) + } } /// A `PollSet` wrapper that satisfies `kbpf_basic::PollWaker`, allowing diff --git a/os/StarryOS/kernel/src/ebpf/mod.rs b/os/StarryOS/kernel/src/ebpf/mod.rs index 57216d903c..e861046945 100644 --- a/os/StarryOS/kernel/src/ebpf/mod.rs +++ b/os/StarryOS/kernel/src/ebpf/mod.rs @@ -41,6 +41,7 @@ pub use transform::EbpfKernelAuxiliary; use crate::{ ebpf::{error::BpfResultExt, map::create_map, prog::load_prog}, file::add_file_like, + kprobe::KernelRawMutex, mm::VmBytes, perf::raw_tracepoint::bpf_raw_tracepoint_open, }; @@ -103,37 +104,37 @@ fn handle_prog_load(attr: &bpf_attr) -> AxResult { fn handle_map_update(attr: &bpf_attr) -> AxResult { let arg = BpfMapUpdateArg::from(attr); - bpf_map_update_elem::(arg).into_ax_result()?; + bpf_map_update_elem::(arg).into_ax_result()?; Ok(0) } fn handle_map_lookup(attr: &bpf_attr) -> AxResult { let arg = BpfMapUpdateArg::from(attr); - bpf_lookup_elem::(arg).into_ax_result()?; + bpf_lookup_elem::(arg).into_ax_result()?; Ok(0) } fn handle_map_delete(attr: &bpf_attr) -> AxResult { let arg = BpfMapUpdateArg::from(attr); - bpf_map_delete_elem::(arg).into_ax_result()?; + bpf_map_delete_elem::(arg).into_ax_result()?; Ok(0) } fn handle_map_get_next_key(attr: &bpf_attr) -> AxResult { let arg = BpfMapGetNextKeyArg::from(attr); - bpf_map_get_next_key::(arg).into_ax_result()?; + bpf_map_get_next_key::(arg).into_ax_result()?; Ok(0) } fn handle_map_freeze(attr: &bpf_attr) -> AxResult { let map_fd = unsafe { attr.__bindgen_anon_2.map_fd }; - bpf_map_freeze::(map_fd).into_ax_result()?; + bpf_map_freeze::(map_fd).into_ax_result()?; Ok(0) } fn handle_map_lookup_and_delete(attr: &bpf_attr) -> AxResult { let arg = BpfMapUpdateArg::from(attr); - bpf_map_lookup_and_delete_elem::(arg).into_ax_result()?; + bpf_map_lookup_and_delete_elem::(arg).into_ax_result()?; Ok(0) } diff --git a/os/StarryOS/kernel/src/ebpf/prog.rs b/os/StarryOS/kernel/src/ebpf/prog.rs index 6bfa206277..6a35384cf6 100644 --- a/os/StarryOS/kernel/src/ebpf/prog.rs +++ b/os/StarryOS/kernel/src/ebpf/prog.rs @@ -10,7 +10,7 @@ use axpoll::Pollable; use kbpf_basic::{preprocessor::EbpfPreProcessor, prog::BpfProgMeta}; use crate::{ - ebpf::{map::BpfMap, transform::EbpfKernelAuxiliary}, + ebpf::{KernelRawMutex, map::BpfMap, transform::EbpfKernelAuxiliary}, file::FileLike, }; @@ -92,7 +92,7 @@ impl FileLike for BpfProg { /// [`BpfProg`]. pub fn load_prog(meta: &mut BpfProgMeta) -> kbpf_basic::BpfResult { let insns = meta.take_insns().ok_or(kbpf_basic::BpfError::EINVAL)?; - let preprocessor = EbpfPreProcessor::preprocess::(insns)?; + let preprocessor = EbpfPreProcessor::preprocess::(insns)?; Ok(BpfProg::new( BpfProgMeta { prog_flags: meta.prog_flags, diff --git a/os/StarryOS/kernel/src/ebpf/transform.rs b/os/StarryOS/kernel/src/ebpf/transform.rs index 1cdefdc845..477192cb81 100644 --- a/os/StarryOS/kernel/src/ebpf/transform.rs +++ b/os/StarryOS/kernel/src/ebpf/transform.rs @@ -32,7 +32,7 @@ use kbpf_basic::{ use rbpf::ebpf::Insn; use crate::{ - ebpf::map::BpfMap, + ebpf::{KernelRawMutex, map::BpfMap}, file::get_file_like, mm::{VmBytes, VmBytesMut, vm_load_string}, }; @@ -130,31 +130,31 @@ impl PerCpuVariants for PerCpuVariantsImpl { pub struct EbpfKernelAuxiliary; impl KernelAuxiliaryOps for EbpfKernelAuxiliary { + type MapLock = KernelRawMutex; fn get_unified_map_from_ptr(ptr: *const u8, func: F) -> kbpf_basic::BpfResult where - F: FnOnce(&mut UnifiedMap) -> kbpf_basic::BpfResult, + F: FnOnce(&UnifiedMap) -> kbpf_basic::BpfResult, { // SAFETY: ptr was produced by `Arc::into_raw` in // `get_unified_map_ptr_from_fd`; the caller passes it back here so // we may reconstruct the Arc, run the closure, and re-leak it. let map = unsafe { Arc::from_raw(ptr as *const BpfMap) }; - let mut unified = map.unified_map(); - let ret = func(&mut unified); - drop(unified); + let unified = map.unified_map(); + let ret = func(unified); let _ = Arc::into_raw(map); ret } fn get_unified_map_from_fd(map_fd: u32, func: F) -> kbpf_basic::BpfResult where - F: FnOnce(&mut UnifiedMap) -> kbpf_basic::BpfResult, + F: FnOnce(&UnifiedMap) -> kbpf_basic::BpfResult, { let file = get_file_like(map_fd as _).map_err(|_| BpfError::ENOENT)?; let bpf_map = file .into_any_arc() .downcast::() .map_err(|_| BpfError::EINVAL)?; - let unified = &mut bpf_map.unified_map(); + let unified = bpf_map.unified_map(); func(unified) } @@ -263,10 +263,15 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary { Ok(res_virt) } - fn unmap(virt_addr: usize) { + fn vunmap(vaddr: usize, num_pages: usize) { let kspace = ax_mm::kernel_aspace(); let mut guard = kspace.lock(); - let _ = guard.unmap(VirtAddr::from_usize(virt_addr), PageSize::Size4K as usize); + guard + .unmap( + VirtAddr::from_usize(vaddr), + PageSize::Size4K as usize * num_pages, + ) + .expect("vmunmap failed"); } } diff --git a/os/StarryOS/kernel/src/entry.rs b/os/StarryOS/kernel/src/entry.rs index bdb1a76a3f..ab6bda1c1b 100644 --- a/os/StarryOS/kernel/src/entry.rs +++ b/os/StarryOS/kernel/src/entry.rs @@ -22,19 +22,13 @@ pub fn init(args: &[String], envs: &[String]) { static_keys::global_init(); tracepoint_init().expect("Failed to initialize tracepoints"); - { - // perf kprobe-by-name resolves through the real in-kernel `.kallsyms` - // blob (`pseudofs::proc::KALLSYMS`, built from the `ksym` crate), the - // same table `/proc/kallsyms` exposes — no separate symbol table. - crate::ebpf::init_ebpf(); - crate::perf::perf_event_init(); - } + crate::ebpf::init_ebpf(); + crate::perf::perf_event_init(); + crate::kmod::init_kmod(); - // FIXME: loongarch64 selftest hangs on QEMU; the kprobe crate's loongarch64 - // breakpoint handling needs upstream fixes before selftest can be enabled. - #[cfg(not(target_arch = "loongarch64"))] - crate::kprobe::run_selftest(); + #[cfg(feature = "kprobe_test")] + crate::kprobe::kprobe_test(); pseudofs::mount_all().expect("Failed to mount pseudofs"); spawn_alarm_task(); diff --git a/os/StarryOS/kernel/src/kprobe.rs b/os/StarryOS/kernel/src/kprobe.rs index b1eb774f8a..95a89d417d 100644 --- a/os/StarryOS/kernel/src/kprobe.rs +++ b/os/StarryOS/kernel/src/kprobe.rs @@ -17,15 +17,15 @@ //! - [`handle_breakpoint`]: Entry point for breakpoint exceptions (INT3/EBREAK/BRK) //! - [`handle_debug`]: Entry point for debug exceptions (x86_64 single-step only) -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; -use ax_kspin::RawSpinNoIrq; +use ax_kspin::{RawSpinNoIrq, SpinNoIrq}; use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr, VirtAddrRange}; use ax_runtime::hal::paging::{MappingFlags, PageSize}; use kprobe::{ KprobeAuxiliaryOps, KretprobeBuilder, ProbeBuilder, ProbePointList, register_kprobe as kprobe_crate_register_kprobe, - register_kretprobe as kprobe_crate_register_kretprobe, + register_kretprobe as kprobe_crate_register_kretprobe, retprobe::RetprobeInstance, unregister_kprobe as kprobe_crate_unregister_kprobe, unregister_kretprobe as kprobe_crate_unregister_kretprobe, }; @@ -118,37 +118,8 @@ impl KprobeAuxiliaryOps for KernelKprobeOps { return; } let addr = VirtAddr::from(address); - let aligned_addr = addr.align_down_4k(); - let aligned_end = (addr + len).align_up_4k(); - let aligned_length: usize = aligned_end - aligned_addr; - - crate::stop_machine::stop_machine( - move || { - let mut guard = ax_mm::kernel_aspace().lock(); - let (_, original_flags, _) = guard - .page_table() - .query(aligned_addr) - .expect("kprobe: set_writeable: address not mapped"); - guard - .protect( - aligned_addr, - aligned_length, - original_flags | MappingFlags::WRITE, - ) - .expect("kprobe: set_writeable: protect failed"); - crate::mm::flush_tlb_range(aligned_addr, aligned_length); - action(addr.as_mut_ptr()); - #[cfg(target_arch = "aarch64")] - ax_runtime::hal::cpu::asm::clean_dcache_range_to_pou(addr, len); - guard - .protect(aligned_addr, aligned_length, original_flags) - .expect("kprobe: set_writeable: restore failed"); - }, - move || { - crate::mm::flush_tlb_range(aligned_addr, aligned_length); - ax_runtime::hal::cpu::asm::flush_icache_all(); - }, - ); + crate::mm::patch_kernel_text(addr, len, action) + .expect("kprobe: set_writeable: patch kernel text failed"); } fn alloc_kernel_exec_memory() -> *mut u8 { @@ -218,18 +189,35 @@ impl KprobeAuxiliaryOps for KernelKprobeOps { .expect("uprobe: unmap user exec memory failed"); } - fn insert_kretprobe_instance_to_task(instance: kprobe::retprobe::RetprobeInstance) { - let curr = ax_task::current(); - curr.as_thread().kretprobe_stack.lock().push(instance); + fn insert_kretprobe_instance_to_task(instance: RetprobeInstance) { + let task = ax_task::current_may_uninit(); + if let Some(task) = task { + let thread = task.try_as_thread(); + if let Some(thread) = thread { + let mut kretprobe_instances = thread.kretprobe_stack.lock(); + kretprobe_instances.push(instance); + return; + } + } + // If the current task is None, we can store it in a static variable + let mut instances = INSTANCE.lock(); + instances.push(instance); } - fn pop_kretprobe_instance_from_task() -> kprobe::retprobe::RetprobeInstance { - let curr = ax_task::current(); - curr.as_thread() - .kretprobe_stack - .lock() - .pop() - .expect("kretprobe instance stack underflow") + fn pop_kretprobe_instance_from_task() -> RetprobeInstance { + let task = ax_task::current_may_uninit(); + if let Some(task) = task { + let thread = task.try_as_thread(); + if let Some(thread) = thread { + let mut kretprobe_instances = thread.kretprobe_stack.lock(); + return kretprobe_instances + .pop() + .expect("kretprobe instance stack underflow"); + } + } + // If the current task is None, we can pop it from the static variable + let mut instances = INSTANCE.lock(); + instances.pop().unwrap() } } @@ -244,40 +232,30 @@ pub type KernelKretprobe = kprobe::Kretprobe; /// The `KprobeAuxiliaryOps` impl, aliased under the name the perf module uses. pub type KprobeAuxiliary = KernelKprobeOps; -static KPROBE_MANAGER: ax_sync::spin::SpinNoIrq> = - ax_sync::spin::SpinNoIrq::new(None); -static KPROBE_POINT_LIST: ax_sync::spin::SpinNoIrq> = - ax_sync::spin::SpinNoIrq::new(None); +static KPROBE_MANAGER: KprobeManager = KprobeManager::new(); +static KPROBE_POINT_LIST: SpinNoIrq = SpinNoIrq::new(KprobePointList::new()); +static INSTANCE: SpinNoIrq> = SpinNoIrq::new(Vec::new()); fn with_manager(f: F) -> R where - F: FnOnce(&mut KprobeManager) -> R, + F: FnOnce(&KprobeManager) -> R, { - let mut guard = KPROBE_MANAGER.lock(); - if guard.is_none() { - *guard = Some(KprobeManager::default()); - } - f(guard.as_mut().expect("kprobe: manager not initialized")) + f(&KPROBE_MANAGER) } fn with_manager_and_list(f: F) -> R where - F: FnOnce(&mut KprobeManager, &mut KprobePointList) -> R, + F: FnOnce(&KprobeManager, &mut KprobePointList) -> R, { - let mut mgr = KPROBE_MANAGER.lock(); - if mgr.is_none() { - *mgr = Some(KprobeManager::default()); - } - let mut list = KPROBE_POINT_LIST.lock(); - if list.is_none() { - *list = Some(KprobePointList::new()); - } - f(mgr.as_mut().unwrap(), list.as_mut().unwrap()) + let mut list = KPROBE_POINT_LIST.try_lock().unwrap(); + f(&KPROBE_MANAGER, &mut list) } /// Register a kprobe into the global manager, returning the live handle. pub fn register_kprobe(builder: ProbeBuilder) -> Arc { - with_manager_and_list(|mgr, list| kprobe_crate_register_kprobe(mgr, list, builder)) + with_manager_and_list(|mgr, list| { + kprobe_crate_register_kprobe(mgr, list, builder).expect("Failed to register kprobe") + }) } /// Unregister a previously registered kprobe. @@ -287,7 +265,9 @@ pub fn unregister_kprobe(kprobe: Arc) { /// Register a kretprobe and return its live handle. pub fn register_kretprobe(builder: KretprobeBuilder) -> Arc { - with_manager_and_list(|mgr, list| kprobe_crate_register_kretprobe(mgr, list, builder)) + with_manager_and_list(|mgr, list| { + kprobe_crate_register_kretprobe(mgr, list, builder).expect("Failed to register kretprobe") + }) } /// Unregister a previously registered kretprobe. @@ -314,7 +294,7 @@ pub(crate) fn trapframe_to_ptregs(tf: &ax_runtime::hal::cpu::TrapFrame) -> kprob rdx: tf.rdx as usize, rsi: tf.rsi as usize, rdi: tf.rdi as usize, - orig_rax: 0, + orig_rax: tf.vector as usize, rip: tf.rip as usize, cs: tf.cs as usize, rflags: tf.rflags as usize, @@ -412,7 +392,7 @@ pub(crate) fn trapframe_to_ptregs(tf: &ax_runtime::hal::cpu::TrapFrame) -> kprob tf.regs.s7, tf.regs.s8, ], - orig_a0: tf.regs.a0, + orig_a0: 0, csr_era: tf.era, csr_badvaddr: 0, csr_crmd: 0, @@ -444,6 +424,7 @@ pub(crate) fn ptregs_write_back(pt: &kprobe::PtRegs, tf: &mut ax_runtime::hal::c tf.rdi = pt.rdi as u64; tf.rip = pt.rip as u64; tf.cs = pt.cs as u64; + tf.vector = pt.orig_rax as u64; tf.rflags = pt.rflags as u64; tf.rsp = pt.rsp as u64; tf.ss = pt.ss as u64; @@ -535,96 +516,124 @@ pub fn handle_breakpoint(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool { ptregs_write_back(&pt_regs, tf); return true; } - // Not a kernel kprobe — it may be a uprobe `int3` planted in the current - // process' user text. The uprobe registry is per-process. - crate::uprobe::break_uprobe_handler(tf).is_some() + false } -#[cfg(not(target_arch = "loongarch64"))] -#[inline(never)] -fn kprobe_selftest_target() -> i32 { - 42 -} - -#[cfg(not(target_arch = "loongarch64"))] -static SELFTEST_HIT: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); -#[cfg(not(target_arch = "loongarch64"))] -static SELFTEST_RET_HIT: core::sync::atomic::AtomicBool = - core::sync::atomic::AtomicBool::new(false); - -#[cfg(not(target_arch = "loongarch64"))] -fn selftest_pre_handler(_data: &dyn kprobe::ProbeData, _pt: &mut kprobe::PtRegs) { - SELFTEST_HIT.store(true, core::sync::atomic::Ordering::SeqCst); +#[cfg(target_arch = "x86_64")] +pub fn handle_debug(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool { + let mut pt_regs = trapframe_to_ptregs(tf); + let handled = with_manager(|manager| kprobe::kprobe_handler_from_debug(manager, &mut pt_regs)); + if handled.is_some() { + ptregs_write_back(&pt_regs, tf); + return true; + } + false } -#[cfg(not(target_arch = "loongarch64"))] -fn selftest_ret_handler(_data: &dyn kprobe::ProbeData, _pt: &mut kprobe::PtRegs) { - SELFTEST_RET_HIT.store(true, core::sync::atomic::Ordering::SeqCst); -} +#[cfg(feature = "kprobe_test")] +mod kprobe_test { + use alloc::string::ToString; -#[cfg(not(target_arch = "loongarch64"))] -pub fn run_selftest() -> bool { - SELFTEST_HIT.store(false, core::sync::atomic::Ordering::SeqCst); - SELFTEST_RET_HIT.store(false, core::sync::atomic::Ordering::SeqCst); + use kprobe::{KretprobeBuilder, ProbeBuilder, ProbeData, PtRegs}; - let target_addr = kprobe_selftest_target as *const () as usize; - let mut kprobe_ok = false; - let mut kretprobe_ok = false; + use crate::kprobe::{register_kprobe, unregister_kprobe}; + #[inline(never)] + #[unsafe(no_mangle)] + fn detect_func(x: usize, y: usize, z: Option) -> Option { + let hart = 0; + ax_println!("detect_func: hart_id: {}, x: {}, y:{}", hart, x, y); + z.map(|z| x + y + z) + } - with_manager(|manager| { - let mut probe_list = kprobe::ProbePointList::new(); - let builder = kprobe::ProbeBuilder::::new() - .with_symbol_addr(target_addr) - .with_pre_handler(selftest_pre_handler) - .with_enable(true); + fn pre_handler(_data: &dyn ProbeData, pt_regs: &mut PtRegs) { + ax_println!( + "[kprobe] pre_handler: arg0: {}, arg1: {}, arg2: {}", + pt_regs.args()[0], + pt_regs.args()[1], + pt_regs.args()[2] + ); + } - let kp = kprobe::register_kprobe(manager, &mut probe_list, builder); - let val = kprobe_selftest_target(); - kprobe_ok = SELFTEST_HIT.load(core::sync::atomic::Ordering::SeqCst); + fn post_handler(_data: &dyn ProbeData, pt_regs: &mut PtRegs) { + ax_println!( + "[kprobe] post_handler: arg0: {}, arg1: {}, arg2: {}", + pt_regs.args()[0], + pt_regs.args()[1], + pt_regs.args()[2] + ); + } - kprobe::unregister_kprobe(manager, &mut probe_list, kp); + fn kret_post_handler(_data: &dyn ProbeData, pt_regs: &mut PtRegs) { + ax_println!( + "[kretprobe] post_handler: ret_value(a0): {}, ret_value(a1): {}", + pt_regs.first_ret_value(), + pt_regs.second_ret_value() + ); + } - if kprobe_ok && val == 42 { - info!("kprobe selftest passed"); - } else { - warn!("kprobe selftest failed: hit={}, val={}", kprobe_ok, val); - } - }); + pub fn kprobe_test() { + ax_println!( + "[kprobe] kprobe test for [detect_func]: {:#x}", + detect_func as *const () as usize + ); + let kprobe_builder = ProbeBuilder::new() + .with_symbol_addr(detect_func as *const () as usize) + .with_offset(0) + .with_enable(true) + .with_pre_handler(pre_handler) + .with_post_handler(post_handler); + + let kprobe = register_kprobe(kprobe_builder); + let new_pre_handler = |_data: &dyn ProbeData, pt_regs: &mut PtRegs| { + ax_println!( + "[kprobe] new_pre_handler: arg0: {}, arg1: {}, arg2: {}", + pt_regs.args()[0], + pt_regs.args()[1], + pt_regs.args()[2] + ); + }; + + let builder2 = ProbeBuilder::new() + .with_symbol("kprobe::detect_func".to_string()) + .with_symbol_addr(detect_func as *const () as usize) + .with_offset(0) + .with_enable(true) + .with_pre_handler(new_pre_handler) + .with_post_handler(post_handler); + + let kprobe2 = register_kprobe(builder2); + ax_println!( + "[kprobe] install 2 kprobes at [detect_func]: {:#x}", + detect_func as *const () as usize + ); - with_manager(|manager| { - let mut probe_list = kprobe::ProbePointList::new(); - let builder = kprobe::KretprobeBuilder::::new(4) - .with_symbol_addr(target_addr) - .with_ret_handler(selftest_ret_handler) - .with_enable(true); + detect_func(1, 2, Some(3)); - let kr = kprobe::register_kretprobe(manager, &mut probe_list, builder); - let val = kprobe_selftest_target(); - kretprobe_ok = SELFTEST_RET_HIT.load(core::sync::atomic::Ordering::SeqCst); + unregister_kprobe(kprobe); + unregister_kprobe(kprobe2); + ax_println!( + "[kprobe] uninstall 2 kprobes at [detect_func]: {:#x}", + detect_func as *const () as usize + ); - kprobe::unregister_kretprobe(manager, &mut probe_list, kr); + let kretprobe_builder = KretprobeBuilder::new(10) + .with_symbol_addr(detect_func as *const () as usize) + .with_enable(true) + .with_ret_handler(kret_post_handler); - if kretprobe_ok && val == 42 { - info!("kretprobe selftest passed"); - } else { - warn!( - "kretprobe selftest failed: hit={}, val={}", - kretprobe_ok, val - ); - } - }); + let kretprobe = crate::kprobe::register_kretprobe(kretprobe_builder); + ax_println!( + "[kretprobe] install kretprobe at [detect_func]: {:#x}", + detect_func as *const () as usize + ); + detect_func(0xff, 0, Some(1)); - kprobe_ok && kretprobe_ok -} + crate::kprobe::unregister_kretprobe(kretprobe); -#[cfg(target_arch = "x86_64")] -pub fn handle_debug(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool { - let mut pt_regs = trapframe_to_ptregs(tf); - let handled = with_manager(|manager| kprobe::kprobe_handler_from_debug(manager, &mut pt_regs)); - if handled.is_some() { - ptregs_write_back(&pt_regs, tf); - return true; + detect_func(3, 4, None); + ax_println!("[kprobe] [kretprobe] test passed"); } - // Fall through to the per-process uprobe single-step (out-of-line) handler. - crate::uprobe::debug_uprobe_handler(tf).is_some() } + +#[cfg(feature = "kprobe_test")] +pub use kprobe_test::kprobe_test; diff --git a/os/StarryOS/kernel/src/mm/access.rs b/os/StarryOS/kernel/src/mm/access.rs index f95cc9ca9d..24fe3dd33b 100644 --- a/os/StarryOS/kernel/src/mm/access.rs +++ b/os/StarryOS/kernel/src/mm/access.rs @@ -515,14 +515,18 @@ impl IoBufMut for VmBytesMut { } } -/// Writes data to kernel text, ensuring the page permissions are properly handled. -pub fn write_kernel_text(addr: VirtAddr, data: &[u8]) -> AxResult<()> { - if data.is_empty() { +/// Patches kernel text, ensuring page permissions and instruction-cache +/// synchronization are handled consistently. +pub fn patch_kernel_text(addr: VirtAddr, len: usize, action: F) -> AxResult<()> +where + F: FnOnce(*mut u8), +{ + if len == 0 { return Ok(()); } let aligned_addr = addr.align_down_4k(); - let aligned_length = (addr + data.len()).align_up_4k() - aligned_addr; + let aligned_length = (addr + len).align_up_4k() - aligned_addr; // The kernel address-space lock (`SpinNoIrq`) MUST be acquired *inside* the // `stop_machine` critical section, not before it. `stop_machine` itself @@ -537,30 +541,52 @@ pub fn write_kernel_text(addr: VirtAddr, data: &[u8]) -> AxResult<()> { crate::stop_machine::stop_machine( move || -> AxResult<()> { let mut guard = ax_mm::kernel_aspace().lock(); - let (_, original_flags, _) = guard.page_table().query(aligned_addr)?; + if guard.contains_range(aligned_addr, aligned_length) { + let (_, original_flags, _) = guard.page_table().query(aligned_addr)?; + + guard.protect( + aligned_addr, + aligned_length, + original_flags | MappingFlags::WRITE, + )?; - guard.protect( - aligned_addr, - aligned_length, - original_flags | MappingFlags::WRITE, - )?; + flush_tlb_range(aligned_addr, aligned_length); + action(addr.as_mut_ptr()); - flush_tlb_range(aligned_addr, aligned_length); + #[cfg(target_arch = "aarch64")] + ax_runtime::hal::cpu::asm::clean_dcache_range_to_pou(addr, len); - unsafe { - core::ptr::copy_nonoverlapping(data.as_ptr(), addr.as_mut_ptr(), data.len()); + guard.protect(aligned_addr, aligned_length, original_flags)?; + return Ok(()); } - #[cfg(target_arch = "aarch64")] - ax_runtime::hal::cpu::asm::clean_dcache_range_to_pou(addr, data.len()); + #[cfg(target_arch = "loongarch64")] + { + // LoongArch64 kernel text may execute from the 0x9000... DMW + // direct-map window. DMW translations do not consult PTEs, so + // there are no page permissions to relax here. Patch directly + // while all other CPUs are parked, then rely on the per-CPU + // sync callback to flush instruction state. + action(addr.as_mut_ptr()); + return Ok(()); + } - guard.protect(aligned_addr, aligned_length, original_flags)?; - Ok(()) + #[cfg(not(target_arch = "loongarch64"))] + { + Err(AxError::BadAddress) + } }, move || sync_modified_kernel_text(aligned_addr, aligned_length), ) } +/// Writes data to kernel text, ensuring the page permissions are properly handled. +pub fn write_kernel_text(addr: VirtAddr, data: &[u8]) -> AxResult<()> { + patch_kernel_text(addr, data.len(), |dst| unsafe { + core::ptr::copy_nonoverlapping(data.as_ptr(), dst, data.len()); + }) +} + pub fn flush_tlb_range(start: VirtAddr, size: usize) { for offset in (0..size).step_by(PAGE_SIZE_4K) { ax_runtime::hal::cpu::asm::flush_tlb(Some(start + offset)); diff --git a/os/StarryOS/kernel/src/mm/aspace/backend/mod.rs b/os/StarryOS/kernel/src/mm/aspace/backend/mod.rs index 848f1c39cd..f9b8003521 100644 --- a/os/StarryOS/kernel/src/mm/aspace/backend/mod.rs +++ b/os/StarryOS/kernel/src/mm/aspace/backend/mod.rs @@ -29,8 +29,10 @@ fn divide_page(size: usize, page_size: PageSize) -> usize { size >> (page_size as usize).trailing_zeros() } -pub(crate) fn alloc_frame(zeroed: bool, size: PageSize) -> AxResult { - let page_size = size as usize; +/// Allocates a single page frame of the given size, returning its physical address. +/// The caller is responsible for deallocating the frame with `dealloc_frame`. +pub(crate) fn alloc_frame(zeroed: bool, page_size: PageSize) -> AxResult { + let page_size = page_size as usize; let num_pages = page_size / PAGE_SIZE_4K; let vaddr = VirtAddr::from( global_allocator() @@ -45,9 +47,10 @@ pub(crate) fn alloc_frame(zeroed: bool, size: PageSize) -> AxResult { Ok(paddr) } -pub(crate) fn dealloc_frame(frame: PhysAddr, align: PageSize) { +/// Deallocates a single page frame previously allocated with `alloc_frame`. +pub(crate) fn dealloc_frame(frame: PhysAddr, page_size: PageSize) { let vaddr = phys_to_virt(frame); - let page_size: usize = align.into(); + let page_size: usize = page_size.into(); let num_pages = page_size / PAGE_SIZE_4K; global_allocator().dealloc_pages(vaddr.as_usize(), num_pages, UsageKind::VirtMem); } diff --git a/os/StarryOS/kernel/src/mm/aspace/backend/shared.rs b/os/StarryOS/kernel/src/mm/aspace/backend/shared.rs index 564ba4e5d5..2551dcfed0 100644 --- a/os/StarryOS/kernel/src/mm/aspace/backend/shared.rs +++ b/os/StarryOS/kernel/src/mm/aspace/backend/shared.rs @@ -1,5 +1,5 @@ use alloc::{sync::Arc, vec::Vec}; -use core::ops::Deref; +use core::{any::Any, ops::Deref}; use ax_errno::AxResult; use ax_memory_addr::{MemoryAddr, PhysAddr, VirtAddr, VirtAddrRange}; @@ -8,9 +8,15 @@ use ax_sync::Mutex; use super::{AddrSpace, Backend, BackendOps, alloc_frame, dealloc_frame, divide_page, pages_in}; +enum SharedPagesOwner { + Allocated, + Borrowed(Option>), +} + pub struct SharedPages { - pub phys_pages: Vec, + phys_pages: Vec, pub size: PageSize, + owner: SharedPagesOwner, } impl SharedPages { pub fn new(size: usize, page_size: PageSize) -> AxResult { @@ -18,6 +24,7 @@ impl SharedPages { let mut result = Self { phys_pages: Vec::with_capacity(num_pages), size: page_size, + owner: SharedPagesOwner::Allocated, }; for _ in 0..num_pages { result.phys_pages.push(alloc_frame(true, page_size)?); @@ -25,6 +32,21 @@ impl SharedPages { Ok(result) } + pub fn borrowed( + phys_pages: Vec, + page_size: PageSize, + retain: Option>, + ) -> AxResult { + if phys_pages.is_empty() { + return Err(ax_errno::AxError::InvalidInput); + } + Ok(Self { + phys_pages, + size: page_size, + owner: SharedPagesOwner::Borrowed(retain), + }) + } + pub fn len(&self) -> usize { self.phys_pages.len() } @@ -44,8 +66,13 @@ impl Deref for SharedPages { impl Drop for SharedPages { fn drop(&mut self) { - for frame in &self.phys_pages { - dealloc_frame(*frame, self.size); + match &self.owner { + SharedPagesOwner::Allocated => { + for frame in &self.phys_pages { + dealloc_frame(*frame, self.size); + } + } + SharedPagesOwner::Borrowed(_retain) => {} } } } diff --git a/os/StarryOS/kernel/src/perf/kprobe.rs b/os/StarryOS/kernel/src/perf/kprobe.rs index 352e17a61b..c66d3cad18 100644 --- a/os/StarryOS/kernel/src/perf/kprobe.rs +++ b/os/StarryOS/kernel/src/perf/kprobe.rs @@ -5,7 +5,7 @@ //! Symbol resolution goes through the real in-kernel `.kallsyms` blob //! (`crate::pseudofs::proc::KALLSYMS`), the same table `/proc/kallsyms` reads. -use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use alloc::{sync::Arc, vec::Vec}; use core::{ any::Any, sync::atomic::{AtomicU32, Ordering}, @@ -120,7 +120,7 @@ impl PerfEventOps for ProbePerfEvent { static CALLBACK_ID: AtomicU32 = AtomicU32::new(0); let id = CALLBACK_ID.fetch_add(1, Ordering::Relaxed); - let callback = Box::new(KprobePerfCallBack::new(vm)); + let callback = Arc::new(KprobePerfCallBack::new(vm)); match self.probe { ProbeTy::Kprobe(ref k) => k.register_event_callback(id, callback), ProbeTy::Kretprobe(ref k) => k.register_event_callback(id, callback), diff --git a/os/StarryOS/kernel/src/pseudofs/device.rs b/os/StarryOS/kernel/src/pseudofs/device.rs index eb98bea7f2..f5d37f077f 100644 --- a/os/StarryOS/kernel/src/pseudofs/device.rs +++ b/os/StarryOS/kernel/src/pseudofs/device.rs @@ -1,8 +1,8 @@ -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use core::{any::Any, task::Context}; use ax_fs::CachedFile; -use ax_memory_addr::PhysAddrRange; +use ax_memory_addr::{PhysAddr, PhysAddrRange}; use axfs_ng_vfs::{ DeviceId, FileNodeOps, FilesystemOps, Metadata, MetadataUpdate, NodeFlags, NodeOps, NodePermission, NodeType, VfsError, VfsResult, @@ -28,6 +28,12 @@ pub enum DeviceMmap { /// This is for file descriptors whose mmap offset is a selector rather than /// a byte offset into a linear device, such as io_uring ring offsets. PhysicalResolved(PhysAddrRange, Option>), + /// Maps to an explicit physical page list for this exact mmap request. + /// The producer has already applied the requested offset and length, so + /// mmap callers must map these pages in order without adding the offset + /// again. This covers layouts that are not a single contiguous physical + /// range, such as BPF ringbuf maps that expose mirrored data pages. + PhysicalPages(Vec, Option>), /// Maps to a cached file. Cache(CachedFile), } diff --git a/os/StarryOS/kernel/src/pseudofs/sysfs.rs b/os/StarryOS/kernel/src/pseudofs/sysfs.rs index 28a3803408..5de82626f5 100644 --- a/os/StarryOS/kernel/src/pseudofs/sysfs.rs +++ b/os/StarryOS/kernel/src/pseudofs/sysfs.rs @@ -573,9 +573,9 @@ impl SimpleDirOps for SystemCpuEntryDir { fn cpu_range_string() -> String { let cpu_num = ax_runtime::hal::cpu_num(); if cpu_num <= 1 { - "0".to_owned() + "0\n".to_owned() } else { - format!("0-{}", cpu_num - 1) + format!("0-{}\n", cpu_num - 1) } } diff --git a/os/StarryOS/kernel/src/syscall/mm/mmap.rs b/os/StarryOS/kernel/src/syscall/mm/mmap.rs index 4b39672347..a6a3d801a3 100644 --- a/os/StarryOS/kernel/src/syscall/mm/mmap.rs +++ b/os/StarryOS/kernel/src/syscall/mm/mmap.rs @@ -216,6 +216,7 @@ pub fn sys_mmap( { Ok(DeviceMmap::Physical(..)) | Ok(DeviceMmap::PhysicalResolved(..)) + | Ok(DeviceMmap::PhysicalPages(..)) | Ok(DeviceMmap::Cache(_)) => false, Ok(DeviceMmap::None) | Err(_) => true, } @@ -351,6 +352,13 @@ pub fn sys_mmap( None => Backend::new_linear(start, pa_va_offset, true), } } + Ok(DeviceMmap::PhysicalPages(pages, retain)) => { + length = length.min(pages.len() * PAGE_SIZE_4K); + Backend::new_shared( + start, + Arc::new(SharedPages::borrowed(pages, PageSize::Size4K, retain)?), + ) + } Ok(DeviceMmap::None) => return Err(AxError::NoSuchDevice), Ok(_) => return Err(AxError::InvalidInput), Err(_) => { @@ -427,6 +435,17 @@ pub fn sys_mmap( None => Backend::new_linear(start, pa_va_offset, true), } } + DeviceMmap::PhysicalPages(pages, retain) => { + length = length.min(pages.len() * PAGE_SIZE_4K); + Backend::new_shared( + start, + Arc::new(SharedPages::borrowed( + pages, + PageSize::Size4K, + retain, + )?), + ) + } DeviceMmap::Cache(cache) => Backend::new_file( start, cache, diff --git a/os/StarryOS/kernel/src/syscall/task/clone.rs b/os/StarryOS/kernel/src/syscall/task/clone.rs index 0998f578d8..ca3eb5aa8c 100644 --- a/os/StarryOS/kernel/src/syscall/task/clone.rs +++ b/os/StarryOS/kernel/src/syscall/task/clone.rs @@ -415,6 +415,30 @@ impl CloneArgs { } } +ktracepoint::define_event_trace!( + sys_clone, + TP_kops(crate::tracepoint::KernelTraceAux), + TP_system(syscalls), + TP_PROTO(flags:u32, stack:usize, parent_tid:usize), + TP_STRUCT__entry { + stack: usize, + parent_tid: usize, + flags: u32, + }, + TP_fast_assign { + flags: flags, + stack: stack, + parent_tid: parent_tid, + }, + TP_ident(__entry), + TP_printk({ + let flags = __entry.flags; + let stack = __entry.stack; + let parent_tid = __entry.parent_tid; + alloc::format!("clone with flags: {flags}, stack: {stack:#x}, parent_tid: {parent_tid:#x}") + }) +); + pub fn sys_clone( uctx: &UserContext, flags: u32, @@ -428,6 +452,8 @@ pub fn sys_clone( let clone_flags = CloneFlags::from_bits_truncate((flags & !FLAG_MASK) as u64); let exit_signal = (flags & FLAG_MASK) as u64; + trace_sys_clone(clone_flags.bits() as _, stack, parent_tid); + if clone_flags.contains(CloneFlags::PIDFD | CloneFlags::PARENT_SETTID) { return Err(AxError::InvalidInput); } diff --git a/os/StarryOS/kernel/src/task/mod.rs b/os/StarryOS/kernel/src/task/mod.rs index 560a99e773..d692fc132f 100644 --- a/os/StarryOS/kernel/src/task/mod.rs +++ b/os/StarryOS/kernel/src/task/mod.rs @@ -578,14 +578,9 @@ pub struct ProcessData { /// The virtual memory address space. // TODO: scopify aspace: SpinNoIrq>>, - /// Per-process uprobe manager. Uprobes plant an `int3` in *this* process' - /// user text, so (unlike the global kprobe manager) the registry is - /// per-address-space. A *sleeping* mutex, because arming/disarming - /// manipulates the user address space (page-table query, faulting reads, - /// mapping the out-of-line single-step page) which requires sleeping locks; - /// the exception-context breakpoint/debug handlers acquire it with - /// `try_lock()` (a single CAS, safe in atomic context) instead. - pub uprobe_manager: Mutex, + /// The per-process uprobe manager. Each process has its own because user + /// code can be modified independently. + pub uprobe_manager: crate::kprobe::KprobeManager, /// Per-process uprobe point list, paired with [`Self::uprobe_manager`]. pub uprobe_point_list: Mutex, /// The resource scope @@ -744,7 +739,7 @@ impl ProcessData { cmdline: RwLock::new(image.cmdline), auxv: RwLock::new(image.auxv), aspace: SpinNoIrq::new(aspace), - uprobe_manager: Mutex::new(crate::kprobe::KprobeManager::default()), + uprobe_manager: crate::kprobe::KprobeManager::new(), uprobe_point_list: Mutex::new(crate::kprobe::KprobePointList::new()), scope: RwLock::new(Scope::new()), heap_top: AtomicUsize::new(crate::config::USER_HEAP_BASE), diff --git a/os/StarryOS/kernel/src/uprobe/mod.rs b/os/StarryOS/kernel/src/uprobe/mod.rs index 6ab2903c38..d2ef25d568 100644 --- a/os/StarryOS/kernel/src/uprobe/mod.rs +++ b/os/StarryOS/kernel/src/uprobe/mod.rs @@ -32,18 +32,18 @@ pub type KernelUprobe = Uprobe; pub fn register_uprobe(builder: ProbeBuilder) -> Arc { let curr = current(); let thread = curr.as_thread(); - let mut manager = thread.proc_data.uprobe_manager.lock(); + let manager = &thread.proc_data.uprobe_manager; let mut point_list = thread.proc_data.uprobe_point_list.lock(); - kprobe::register_uprobe(&mut manager, &mut point_list, builder) + kprobe::register_uprobe(manager, &mut point_list, builder).unwrap() } /// Unregister a previously registered uprobe from the current process. pub fn unregister_uprobe(uprobe: Arc) { let curr = current(); let thread = curr.as_thread(); - let mut manager = thread.proc_data.uprobe_manager.lock(); + let manager = &thread.proc_data.uprobe_manager; let mut point_list = thread.proc_data.uprobe_point_list.lock(); - kprobe::unregister_uprobe(&mut manager, &mut point_list, uprobe); + kprobe::unregister_uprobe(manager, &mut point_list, uprobe); } /// Dispatch a breakpoint exception to the current process' uprobe manager. @@ -56,9 +56,9 @@ pub fn unregister_uprobe(uprobe: Arc) { /// so the lock is always acquired; a contended miss just reports "unhandled". pub fn break_uprobe_handler(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> Option<()> { let curr = current(); - let mut manager = curr.as_thread().proc_data.uprobe_manager.try_lock()?; + let manager = &curr.as_thread().proc_data.uprobe_manager; let mut pt_regs = trapframe_to_ptregs(tf); - let res = kprobe::uprobe_handler_from_break(&mut manager, &mut pt_regs); + let res = kprobe::uprobe_handler_from_break(manager, &mut pt_regs); ptregs_write_back(&pt_regs, tf); res } @@ -68,11 +68,9 @@ pub fn break_uprobe_handler(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> Option< #[cfg(target_arch = "x86_64")] pub fn debug_uprobe_handler(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> Option<()> { let curr = current(); - // `try_lock()` for the same reason as `break_uprobe_handler`: exception - // context, sleeping mutex. - let mut manager = curr.as_thread().proc_data.uprobe_manager.try_lock()?; + let manager = &curr.as_thread().proc_data.uprobe_manager; let mut pt_regs = trapframe_to_ptregs(tf); - let res = kprobe::uprobe_handler_from_debug(&mut manager, &mut pt_regs); + let res = kprobe::uprobe_handler_from_debug(manager, &mut pt_regs); ptregs_write_back(&pt_regs, tf); res } diff --git a/os/arceos/modules/axmm/src/lib.rs b/os/arceos/modules/axmm/src/lib.rs index 1a525b1ef4..f9af4942ff 100644 --- a/os/arceos/modules/axmm/src/lib.rs +++ b/os/arceos/modules/axmm/src/lib.rs @@ -16,7 +16,7 @@ use ax_hal::{ }; use ax_kspin::SpinNoIrq; use ax_lazyinit::LazyInit; -use ax_memory_addr::{MemoryAddr, PhysAddr, VirtAddr}; +use ax_memory_addr::{MemoryAddr, PhysAddr, VirtAddr, VirtAddrRange}; pub use self::{aspace::AddrSpace, backend::Backend}; @@ -63,12 +63,17 @@ pub fn new_kernel_aspace() -> AxResult { // mapped range should contain the whole region if it is not aligned. let start = r.paddr.align_down_4k(); let end = (r.paddr + r.size).align_up_4k(); - aspace.map_linear( - phys_to_virt(start), - start, - end - start, - reg_flag_to_map_flag(r.flags), - )?; + let vaddr = phys_to_virt(start); + let size = end - start; + + // Some platforms, such as LoongArch64 with DMW enabled, provide the + // physical direct map outside the page-table-backed kernel address + // space. Those ranges must not be inserted into the kernel page table: + // DMW accesses do not consult PTEs, and their low VA bits can alias + // real page-table mappings such as vmap. + if aspace.contains_range(vaddr, size) { + aspace.map_linear(vaddr, start, size, reg_flag_to_map_flag(r.flags))?; + } } Ok(aspace) } @@ -114,17 +119,33 @@ pub fn iomap(addr: PhysAddr, size: usize) -> AxResult { let virt_aligned = virt.align_down_4k(); let addr_aligned = addr.align_down_4k(); let size_aligned = (addr + size).align_up_4k() - addr_aligned; + let offset = addr - addr_aligned; let flags = MappingFlags::DEVICE | MappingFlags::READ | MappingFlags::WRITE; let mut tb = kernel_aspace().lock(); - match tb.map_linear(virt_aligned, addr_aligned, size_aligned, flags) { - Err(AxError::AlreadyExists) => { - tb.map_linear_overwrite(virt_aligned, addr_aligned, size_aligned, flags)?; - } - Err(e) => { - return Err(e); + + let mapped = if tb.contains_range(virt_aligned, size_aligned) { + match tb.map_linear(virt_aligned, addr_aligned, size_aligned, flags) { + Err(AxError::AlreadyExists) => { + tb.map_linear_overwrite(virt_aligned, addr_aligned, size_aligned, flags)?; + } + Err(e) => { + return Err(e); + } + Ok(_) => {} } - Ok(_) => {} - } - Ok(virt) + virt_aligned + } else { + // On platforms where `phys_to_virt()` is a hardware direct map outside + // the page-table-backed kernel address space, such as LoongArch64 DMW, + // allocate a separate kernel VA and map the device with PTE attributes. + let range = VirtAddrRange::new(tb.base(), tb.end()); + let mapped = tb + .find_free_area(tb.base(), size_aligned, range) + .ok_or(AxError::NoMemory)?; + tb.map_linear(mapped, addr_aligned, size_aligned, flags)?; + mapped + }; + + Ok(mapped + offset) } diff --git a/platforms/ax-plat-loongarch64-qemu-virt/axconfig.toml b/platforms/ax-plat-loongarch64-qemu-virt/axconfig.toml index 7518e2adda..25b99dd848 100644 --- a/platforms/ax-plat-loongarch64-qemu-virt/axconfig.toml +++ b/platforms/ax-plat-loongarch64-qemu-virt/axconfig.toml @@ -34,10 +34,15 @@ phys-bus-offset = 0 # uint kernel-base-paddr = 0x0020_0000 # uint # Base virtual address of the kernel image. kernel-base-vaddr = "0x9000_0000_0020_0000" # uint -# Kernel address space base. -kernel-aspace-base = "0x9000_0000_0000_0000" # uint -# Kernel address space size. -kernel-aspace-size = "0x0000_ffff_ffff_f000" # uint +# Kernel page-table-backed address space. +# +# LoongArch64 uses the 0x9000_0000_0000_0000 DMW window for cached direct +# mapping, so the kernel page table must not manage that range. Use a +# sign-extended PGDH address whose low 48-bit page-table indexes do not alias +# the DMW mappings. +kernel-aspace-base = "0xFFFF_8000_0000_0000" # uint +# Kernel page-table-backed address space size. +kernel-aspace-size = "0x0000_7fff_ffff_f000" # uint # Stack size on bootstrapping. (256K) boot-stack-size = 0x40000 # uint diff --git a/scripts/axbuild/scripts/starry-kallsyms.sh b/scripts/axbuild/scripts/starry-kallsyms.sh index 80f2778e84..faa8713cde 100644 --- a/scripts/axbuild/scripts/starry-kallsyms.sh +++ b/scripts/axbuild/scripts/starry-kallsyms.sh @@ -89,7 +89,7 @@ pad_kallsyms_to_section() { fi if [ "$kallsyms_size" -lt "$section_size" ]; then - dd if=/dev/zero bs=1 count=$((section_size - kallsyms_size)) >> "$kallsyms" 2>/dev/null + truncate -s "$section_size" "$kallsyms" fi }