fix(starry): support eBPF ringbuf mmap on LoongArch DMW

Cargo.toml

@@ -8,7 +8,7 @@ keywords = ["arceos", "kernel"]
 categories = ["os", "no-std"]
 
 [profile.release]
-lto = true
+lto = false
 
 [workspace]
 resolver = "3"

os/StarryOS/docs/loongarch64-dmw-kernel-aspace.md

@@ -0,0 +1,43 @@
+# LoongArch64 DMW and Kernel Address Space
+
+LoongArch64 platforms may enable DMW (Direct Mapping Window) for the cached
+physical direct map. In the QEMU virt platform, `phys_to_virt()` uses the
+`0x9000_0000_0000_0000` DMW window:
+
+```text
+VA = 0x9000_0000_0000_0000 + PA
+```
+
+This range is translated by DMW hardware and does not consult the kernel page
+table. Therefore it must not be treated as ordinary page-table-backed kernel
+virtual memory.
+
+The page-table-backed kernel address space should use a non-DMW address range,
+and must still be a legal mapped virtual address. With the current
+LoongArch64 page-table configuration (`VALEN = 48`), bits `[63:48]` must be a
+sign extension of bit 47. For example:
+
+```toml
+kernel-aspace-base = "0xFFFF_8000_0000_0000"
+kernel-aspace-size = "0x0000_7fff_ffff_f000"
+```
+
+Temporary kernel mappings such as `vmap`, eBPF ring-buffer aliases, MMIO
+`iomap` ranges, module memory, and trampoline pages should be allocated from
+this page-table-backed kernel address space.
+
+Do not add a second page-table mapping for DMW direct-map RAM. Besides being
+redundant, it can conflict with real kernel virtual mappings because the current
+LoongArch64 page-table implementation indexes only the low 48 virtual-address
+bits. For example, mappings in `0x9000_...` and page-table-backed mappings can
+share the same page-table indexes if their low 48 bits are equal.
+
+`ax-mm` therefore maps a physical memory region through `phys_to_virt()` only
+when the resulting virtual range is contained in the configured kernel address
+space. On DMW platforms this skips the hardware direct-map range; on platforms
+where the direct map is page-table-backed, the existing mapping behavior is
+preserved.
+
+
+## References
+https://www.kernel.org/doc/html/v6.1/loongarch/introduction.html#virtual-memory

os/StarryOS/kernel/Cargo.toml

@@ -16,6 +16,7 @@ license.workspace = true
 [features]
 default = ["dynamic_debug"]
 dev-log = []
+kprobe_test = []
 ext4 = ["ax-fs/ext4"]
 input = ["dep:ax-input", "ax-feat/input"]
 memtrack = ["ax-feat/backtrace", "ax-alloc/tracking"]
@@ -134,7 +135,7 @@ zerocopy = { version = "0.8", features = ["derive"] }
 ax-ipi = { workspace = true }
 static-keys = "0.8"
 ddebug = "0.5"
-kprobe = "0.5"
+kprobe = "0.6"
 kmod = { version = "0.2", package = "kmod-tools" }
 kmod-loader = "0.2.1"
 lwprintf-rs = "0.3"
@@ -145,7 +146,7 @@ sg200x-bsp = { workspace = true, optional = true }
 tock-registers = { version = "0.9", optional = true }
 ktracepoint = "0.6"
 ksym = "0.6"
-kbpf-basic = "0.5.7"
+kbpf-basic = "0.6"
 rbpf = { version = "0.4", default-features = false }
 
 [target.'cfg(target_arch = "x86_64")'.dependencies]

os/StarryOS/kernel/src/ebpf/map.rs

@@ -3,11 +3,11 @@
 //! to tgoskits' `ax_hal` / `ax_kspin` / `ax_errno` / `ax_alloc` package
 //! names per `crate-fork-audit.md §6`.
 
-use alloc::{borrow::Cow, sync::Arc};
+use alloc::{borrow::Cow, sync::Arc, vec::Vec};
 use core::ops::{Deref, DerefMut};
 
 use ax_errno::{AxError, AxResult};
-use ax_kspin::{SpinNoPreempt, SpinNoPreemptGuard};
+use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr};
 use axpoll::{PollSet, Pollable};
 use kbpf_basic::{
     PollWaker,
@@ -17,13 +17,15 @@ use kbpf_basic::{
 use crate::{
     ebpf::transform::{EbpfKernelAuxiliary, PerCpuImpl},
     file::{FileLike, Kstat},
+    kprobe::KernelRawMutex,
+    pseudofs::DeviceMmap,
 };
 
 /// File-like handle for a BPF map. Holds the `UnifiedMap` (the kbpf-basic
 /// abstraction over array / hash / lru / queue / perf-array maps) and a
 /// `PollSet` so `poll(2)`-based maps (e.g. ringbuf) can wake waiters.
 pub struct BpfMap {
-    unified_map: SpinNoPreempt<UnifiedMap>,
+    unified_map: UnifiedMap<KernelRawMutex>,
     poll_ready: Arc<PollSetWrapper>,
 }
 
@@ -35,16 +37,16 @@ impl core::fmt::Debug for BpfMap {
 
 impl BpfMap {
     /// Wrap a freshly-created `UnifiedMap` in the kernel file-like layer.
-    pub fn new(unified_map: UnifiedMap, poll_ready: Arc<PollSetWrapper>) -> Self {
+    pub fn new(unified_map: UnifiedMap<KernelRawMutex>, poll_ready: Arc<PollSetWrapper>) -> Self {
         BpfMap {
-            unified_map: SpinNoPreempt::new(unified_map),
+            unified_map,
             poll_ready,
         }
     }
 
     /// Lock and access the underlying `UnifiedMap`.
-    pub fn unified_map(&self) -> SpinNoPreemptGuard<'_, UnifiedMap> {
-        self.unified_map.lock()
+    pub fn unified_map(&self) -> &UnifiedMap<KernelRawMutex> {
+        &self.unified_map
     }
 }
 
@@ -82,6 +84,27 @@ impl FileLike for BpfMap {
     fn path(&self) -> Cow<'_, str> {
         "anon_inode:[bpf_map]".into()
     }
+
+    fn device_mmap(&self, offset: u64, length: u64) -> AxResult<crate::pseudofs::DeviceMmap> {
+        // for ringbuf maps, userland calls mmap on the map fd to get a pointer to the ringbuf;
+        // the kernel must support this. For other map types, mmap is not meaningful and Linux rejects it with EINVAL.
+        if !offset.is_multiple_of(PAGE_SIZE_4K as u64)
+            || !length.is_multiple_of(PAGE_SIZE_4K as u64)
+        {
+            return Err(AxError::InvalidInput);
+        }
+
+        let unified_map = self.unified_map();
+        let map = unified_map.map();
+        let phy_addrs = map
+            .map_mmap(offset as usize, length as usize)
+            .map_err(|_| AxError::InvalidInput)?
+            .iter()
+            .map(|&phys_addr| PhysAddr::from_usize(phys_addr))
+            .collect::<Vec<_>>();
+
+        Ok(DeviceMmap::PhysicalPages(phy_addrs, None))
+    }
 }
 
 /// A `PollSet` wrapper that satisfies `kbpf_basic::PollWaker`, allowing

os/StarryOS/kernel/src/ebpf/mod.rs

@@ -41,6 +41,7 @@ pub use transform::EbpfKernelAuxiliary;
 use crate::{
     ebpf::{error::BpfResultExt, map::create_map, prog::load_prog},
     file::add_file_like,
+    kprobe::KernelRawMutex,
     mm::VmBytes,
     perf::raw_tracepoint::bpf_raw_tracepoint_open,
 };
@@ -103,37 +104,37 @@ fn handle_prog_load(attr: &bpf_attr) -> AxResult<isize> {
 
 fn handle_map_update(attr: &bpf_attr) -> AxResult<isize> {
     let arg = BpfMapUpdateArg::from(attr);
-    bpf_map_update_elem::<EbpfKernelAuxiliary>(arg).into_ax_result()?;
+    bpf_map_update_elem::<EbpfKernelAuxiliary, KernelRawMutex>(arg).into_ax_result()?;
     Ok(0)
 }
 
 fn handle_map_lookup(attr: &bpf_attr) -> AxResult<isize> {
     let arg = BpfMapUpdateArg::from(attr);
-    bpf_lookup_elem::<EbpfKernelAuxiliary>(arg).into_ax_result()?;
+    bpf_lookup_elem::<EbpfKernelAuxiliary, KernelRawMutex>(arg).into_ax_result()?;
     Ok(0)
 }
 
 fn handle_map_delete(attr: &bpf_attr) -> AxResult<isize> {
     let arg = BpfMapUpdateArg::from(attr);
-    bpf_map_delete_elem::<EbpfKernelAuxiliary>(arg).into_ax_result()?;
+    bpf_map_delete_elem::<EbpfKernelAuxiliary, KernelRawMutex>(arg).into_ax_result()?;
     Ok(0)
 }
 
 fn handle_map_get_next_key(attr: &bpf_attr) -> AxResult<isize> {
     let arg = BpfMapGetNextKeyArg::from(attr);
-    bpf_map_get_next_key::<EbpfKernelAuxiliary>(arg).into_ax_result()?;
+    bpf_map_get_next_key::<EbpfKernelAuxiliary, KernelRawMutex>(arg).into_ax_result()?;
     Ok(0)
 }
 
 fn handle_map_freeze(attr: &bpf_attr) -> AxResult<isize> {
     let map_fd = unsafe { attr.__bindgen_anon_2.map_fd };
-    bpf_map_freeze::<EbpfKernelAuxiliary>(map_fd).into_ax_result()?;
+    bpf_map_freeze::<EbpfKernelAuxiliary, KernelRawMutex>(map_fd).into_ax_result()?;
     Ok(0)
 }
 
 fn handle_map_lookup_and_delete(attr: &bpf_attr) -> AxResult<isize> {
     let arg = BpfMapUpdateArg::from(attr);
-    bpf_map_lookup_and_delete_elem::<EbpfKernelAuxiliary>(arg).into_ax_result()?;
+    bpf_map_lookup_and_delete_elem::<EbpfKernelAuxiliary, KernelRawMutex>(arg).into_ax_result()?;
     Ok(0)
 }
 

os/StarryOS/kernel/src/ebpf/prog.rs

@@ -10,7 +10,7 @@ use axpoll::Pollable;
 use kbpf_basic::{preprocessor::EbpfPreProcessor, prog::BpfProgMeta};
 
 use crate::{
-    ebpf::{map::BpfMap, transform::EbpfKernelAuxiliary},
+    ebpf::{KernelRawMutex, map::BpfMap, transform::EbpfKernelAuxiliary},
     file::FileLike,
 };
 
@@ -92,7 +92,7 @@ impl FileLike for BpfProg {
 /// [`BpfProg`].
 pub fn load_prog(meta: &mut BpfProgMeta) -> kbpf_basic::BpfResult<BpfProg> {
     let insns = meta.take_insns().ok_or(kbpf_basic::BpfError::EINVAL)?;
-    let preprocessor = EbpfPreProcessor::preprocess::<EbpfKernelAuxiliary>(insns)?;
+    let preprocessor = EbpfPreProcessor::preprocess::<EbpfKernelAuxiliary, KernelRawMutex>(insns)?;
     Ok(BpfProg::new(
         BpfProgMeta {
             prog_flags: meta.prog_flags,

os/StarryOS/kernel/src/ebpf/transform.rs

@@ -32,7 +32,7 @@ use kbpf_basic::{
 use rbpf::ebpf::Insn;
 
 use crate::{
-    ebpf::map::BpfMap,
+    ebpf::{KernelRawMutex, map::BpfMap},
     file::get_file_like,
     mm::{VmBytes, VmBytesMut, vm_load_string},
 };
@@ -130,31 +130,31 @@ impl<T: Send + Sync + Clone> PerCpuVariants<T> for PerCpuVariantsImpl<T> {
 pub struct EbpfKernelAuxiliary;
 
 impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
+    type MapLock = KernelRawMutex;
     fn get_unified_map_from_ptr<F, R>(ptr: *const u8, func: F) -> kbpf_basic::BpfResult<R>
     where
-        F: FnOnce(&mut UnifiedMap) -> kbpf_basic::BpfResult<R>,
+        F: FnOnce(&UnifiedMap<Self::MapLock>) -> kbpf_basic::BpfResult<R>,
     {
         // SAFETY: ptr was produced by `Arc::into_raw` in
         // `get_unified_map_ptr_from_fd`; the caller passes it back here so
         // we may reconstruct the Arc, run the closure, and re-leak it.
         let map = unsafe { Arc::from_raw(ptr as *const BpfMap) };
-        let mut unified = map.unified_map();
-        let ret = func(&mut unified);
-        drop(unified);
+        let unified = map.unified_map();
+        let ret = func(unified);
         let _ = Arc::into_raw(map);
         ret
     }
 
     fn get_unified_map_from_fd<F, R>(map_fd: u32, func: F) -> kbpf_basic::BpfResult<R>
     where
-        F: FnOnce(&mut UnifiedMap) -> kbpf_basic::BpfResult<R>,
+        F: FnOnce(&UnifiedMap<Self::MapLock>) -> kbpf_basic::BpfResult<R>,
     {
         let file = get_file_like(map_fd as _).map_err(|_| BpfError::ENOENT)?;
         let bpf_map = file
             .into_any_arc()
             .downcast::<BpfMap>()
             .map_err(|_| BpfError::EINVAL)?;
-        let unified = &mut bpf_map.unified_map();
+        let unified = bpf_map.unified_map();
         func(unified)
     }
 
@@ -263,10 +263,15 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
         Ok(res_virt)
     }
 
-    fn unmap(virt_addr: usize) {
+    fn vunmap(vaddr: usize, num_pages: usize) {
         let kspace = ax_mm::kernel_aspace();
         let mut guard = kspace.lock();
-        let _ = guard.unmap(VirtAddr::from_usize(virt_addr), PageSize::Size4K as usize);
+        guard
+            .unmap(
+                VirtAddr::from_usize(vaddr),
+                PageSize::Size4K as usize * num_pages,
+            )
+            .expect("vmunmap failed");
     }
 }
 

os/StarryOS/kernel/src/entry.rs

@@ -22,19 +22,13 @@ pub fn init(args: &[String], envs: &[String]) {
     static_keys::global_init();
     tracepoint_init().expect("Failed to initialize tracepoints");
 
-    {
-        // perf kprobe-by-name resolves through the real in-kernel `.kallsyms`
-        // blob (`pseudofs::proc::KALLSYMS`, built from the `ksym` crate), the
-        // same table `/proc/kallsyms` exposes — no separate symbol table.
-        crate::ebpf::init_ebpf();
-        crate::perf::perf_event_init();
-    }
+    crate::ebpf::init_ebpf();
+    crate::perf::perf_event_init();
+
     crate::kmod::init_kmod();
 
-    // FIXME: loongarch64 selftest hangs on QEMU; the kprobe crate's loongarch64
-    // breakpoint handling needs upstream fixes before selftest can be enabled.
-    #[cfg(not(target_arch = "loongarch64"))]
-    crate::kprobe::run_selftest();
+    #[cfg(feature = "kprobe_test")]
+    crate::kprobe::kprobe_test();
 
     pseudofs::mount_all().expect("Failed to mount pseudofs");
     spawn_alarm_task();