feat(starry-kernel): eBPF kernel runtime (tracepoint / kprobe / perf)

os/StarryOS/kernel/Cargo.toml

@@ -47,6 +47,7 @@ ax-feat = { workspace = true, features = [
 
     "multitask",
     "task-ext",
+    "tracepoint-hooks",
     "sched-rr",
 
     "rtc",
@@ -57,9 +58,11 @@ ax-feat = { workspace = true, features = [
 
 ax-alloc = { workspace = true }
 ax-config.workspace = true
+ax-crate-interface.workspace = true
 ax-display.workspace = true
 ax-fs = { version = "0.5.16", path = "../../arceos/modules/axfs-ng", package = "ax-fs-ng" }
 ax-cpu = { workspace = true, features = ["fp-simd"] }
+ax-hal.workspace = true
 ax-input = { workspace = true, optional = true }
 ax-lazyinit.workspace = true
 ax-log.workspace = true

os/StarryOS/kernel/src/ebpf/map.rs

@@ -43,6 +43,12 @@ impl BpfMap {
     }
 
     /// Lock and access the underlying `UnifiedMap`.
+    ///
+    /// `#[inline(never)]`: a loadable `kebpf.ko` reaches the map storage held
+    /// in *this* kernel's fd table through this accessor and relocates against
+    /// it by name via `.kallsyms`; keep it standalone (see the note in
+    /// `ebpf::transform`).
+    #[inline(never)]
     pub fn unified_map(&self) -> SpinNoPreemptGuard<'_, UnifiedMap> {
         self.unified_map.lock()
     }

os/StarryOS/kernel/src/ebpf/mod.rs

@@ -54,7 +54,15 @@ pub static BPF_HELPER_FUN_SET: LazyInit<BTreeMap<u32, RawBPFHelperFn>> = LazyIni
 /// `kbpf-basic`. Must be called before `sys_bpf(BPF_PROG_LOAD)` so loaded
 /// programs can resolve the helper ids referenced in their instructions.
 pub fn init_ebpf() {
-    let set = kbpf_basic::helper::init_helper_functions::<EbpfKernelAuxiliary>();
+    let mut set = kbpf_basic::helper::init_helper_functions::<EbpfKernelAuxiliary>();
+    // aya emits `bpf_probe_read_kernel` (helper id 113) for reads of kernel
+    // context memory — e.g. `TracePointContext::read_at`, which a cooked
+    // tracepoint program uses to pull fields out of its sample buffer.
+    // kbpf-basic only registers the legacy `bpf_probe_read` (id 4), whose raw
+    // reader is address-space-agnostic in this VM, so alias 113 onto it.
+    if let Some(&probe_read) = set.get(&4) {
+        set.entry(113).or_insert(probe_read);
+    }
     BPF_HELPER_FUN_SET.init_once(set);
 }
 
@@ -86,6 +94,14 @@ fn handle_map_create(attr: &bpf_attr) -> AxResult<isize> {
 
 fn handle_prog_load(attr: &bpf_attr) -> AxResult<isize> {
     let mut meta = BpfProgMeta::try_from_bpf_attr::<EbpfKernelAuxiliary>(attr).into_ax_result()?;
+    // Mirror the loadable `kebpf.ko`'s prog-load path so these `kbpf-basic`
+    // monomorphizations (`BpfProgMeta: Debug`, `BpfProgVerifierInfo:
+    // From<&bpf_attr>`) are instantiated as standalone `.kallsyms` symbols the
+    // module can relocate against. (The module's relocations are resolved at
+    // load time, so they must exist even though the symbols are only referenced
+    // here for that purpose.)
+    debug!("bpf prog load meta: {meta:#?}");
+    let _verifier_info = kbpf_basic::prog::BpfProgVerifierInfo::from(attr);
     let prog = load_prog(&mut meta).into_ax_result()?;
     // bpf prog fds are close-on-exec in Linux as well; see `handle_map_create`.
     let fd = add_file_like(Arc::new(prog), true)?;

os/StarryOS/kernel/src/ebpf/prog.rs

@@ -46,6 +46,10 @@ impl BpfProg {
 }
 
 impl Drop for BpfProg {
+    // `#[inline(never)]`: when a loadable `kebpf.ko` constructs a `BpfProg`, the
+    // resulting fd's drop glue calls this destructor by name through
+    // `.kallsyms`; keep it standalone (see `ebpf::transform`).
+    #[inline(never)]
     fn drop(&mut self) {
         // The preprocessor stashed `Arc<BpfMap>::into_raw` pointers in each
         // map-fd relocated operand so they outlive the load; reconstruct and

os/StarryOS/kernel/src/ebpf/transform.rs

@@ -158,6 +158,18 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
         func(unified)
     }
 
+    // `#[inline(never)]` here (and on the other leaf `KernelAuxiliaryOps`
+    // callbacks below) is load-bearing for the loadable-module path: a
+    // `kebpf.ko` carries its own copy of the stateless `kbpf-basic` map/prog
+    // logic but must call back into *this* kernel's address-space / fd-table /
+    // frame-allocator glue. The module relocates against these methods by their
+    // exact mangled name through `.kallsyms`, so they must survive as standalone
+    // symbols. Without `inline(never)` the optimizer folds them into the
+    // built-in `sys_bpf` call sites and they vanish from the symbol table (the
+    // kallsyms filter keeps even *local* `t` symbols, so non-`pub` is fine —
+    // only inlining, not visibility, removes them). See
+    // `docs/ebpf-followup/syscall-registration.md`.
+    #[inline(never)]
     fn get_unified_map_ptr_from_fd(map_fd: u32) -> kbpf_basic::BpfResult<*const u8> {
         let file = get_file_like(map_fd as _).map_err(|_| BpfError::ENOENT)?;
         let bpf_map = file
@@ -167,6 +179,7 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
         Ok(Arc::into_raw(bpf_map) as *const u8)
     }
 
+    #[inline(never)]
     fn translate_instruction(
         instruction: Vec<u8>,
     ) -> kbpf_basic::BpfResult<Vec<impl kbpf_basic::preprocessor::EbpfInst>> {
@@ -175,6 +188,7 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
         Ok(translated)
     }
 
+    #[inline(never)]
     fn copy_from_user(src: *const u8, size: usize, dst: &mut [u8]) -> kbpf_basic::BpfResult<()> {
         let n = VmBytes::new(src, size)
             .read(dst)
@@ -186,6 +200,7 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
         }
     }
 
+    #[inline(never)]
     fn copy_to_user(dest: *mut u8, size: usize, src: &[u8]) -> kbpf_basic::BpfResult<()> {
         let n = VmBytesMut::new(dest, size)
             .write(src)
@@ -210,6 +225,7 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
         crate::perf::perf_event_output(ctx, fd as usize, flags, data).map_err(|_| BpfError::EINVAL)
     }
 
+    #[inline(never)]
     fn string_from_user_cstr(ptr: *const u8) -> kbpf_basic::BpfResult<String> {
         vm_load_string(ptr as *const _).map_err(|_| BpfError::EFAULT)
     }
@@ -223,6 +239,7 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
         Ok(monotonic_time_nanos())
     }
 
+    #[inline(never)]
     fn alloc_page() -> kbpf_basic::BpfResult<usize> {
         // Reuse the address-space backend's frame allocator
         // (`mm::aspace::backend::alloc_frame`) so eBPF page allocation goes
@@ -232,10 +249,12 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
             .map_err(|_| BpfError::ENOMEM)
     }
 
+    #[inline(never)]
     fn free_page(phys_addr: usize) {
         crate::mm::dealloc_frame(PhysAddr::from_usize(phys_addr), PageSize::Size4K);
     }
 
+    #[inline(never)]
     fn vmap(phys_addrs: &[usize]) -> kbpf_basic::BpfResult<usize> {
         let len = phys_addrs.len() * PageSize::Size4K as usize;
         let kspace = ax_mm::kernel_aspace();
@@ -263,6 +282,7 @@ impl KernelAuxiliaryOps for EbpfKernelAuxiliary {
         Ok(res_virt)
     }
 
+    #[inline(never)]
     fn unmap(virt_addr: usize) {
         let kspace = ax_mm::kernel_aspace();
         let mut guard = kspace.lock();

os/StarryOS/kernel/src/file/ion.rs

@@ -79,7 +79,7 @@ impl FileLike for IonBufferFile {
         Cow::Borrowed("/dev/ion_buffer")
     }
 
-    fn device_mmap(&self, _offset: u64) -> AxResult<DeviceMmap> {
+    fn device_mmap(&self, _offset: u64, _length: u64) -> AxResult<DeviceMmap> {
         Ok(DeviceMmap::Physical(self.phys_range(), None))
     }
 }

os/StarryOS/kernel/src/file/mod.rs

@@ -176,7 +176,7 @@ pub trait FileLike: Pollable + DowncastSync {
         Err(AxError::NoSuchDevice)
     }
 
-    fn device_mmap(&self, _offset: u64) -> AxResult<DeviceMmap> {
+    fn device_mmap(&self, _offset: u64, _length: u64) -> AxResult<DeviceMmap> {
         Err(AxError::BadFileDescriptor)
     }
 

os/StarryOS/kernel/src/kprobe.rs

@@ -49,13 +49,39 @@ pub struct KernelKprobeOps;
 
 impl KprobeAuxiliaryOps for KernelKprobeOps {
     fn copy_memory(src: *const u8, dst: *mut u8, len: usize, user_pid: Option<i32>) {
-        if let Some(_pid) = user_pid {
-            unsafe {
-                let buf =
-                    core::slice::from_raw_parts_mut(dst as *mut core::mem::MaybeUninit<u8>, len);
-                if let Err(e) = starry_vm::vm_read_slice(src, buf) {
-                    warn!("kprobe copy_memory: vm_read_slice failed: {:?}", e);
+        if let Some(pid) = user_pid {
+            // Uprobe arm/disarm reads the target process' original text bytes
+            // while the per-process kprobe manager spin-lock is held (IRQs
+            // disabled), so the faultable user-access path (`vm_read_slice`,
+            // which asserts IRQs enabled) cannot be used. Read through the
+            // *kernel* direct-map alias of the target page's physical frame
+            // instead — the same aliasing `set_writeable_for_address` uses to
+            // write. The text page is already resident (the loader executes the
+            // probed function before arming).
+            let Ok(task) = crate::task::get_task(pid as _) else {
+                warn!("kprobe copy_memory: target task {pid} gone");
+                return;
+            };
+            let aspace = task.as_thread().proc_data.aspace();
+            let mm = aspace.lock();
+            let pt = mm.page_table();
+            let mut copied = 0;
+            while copied < len {
+                let vaddr = VirtAddr::from(src as usize + copied);
+                let Ok((paddr, ..)) = pt.query(vaddr) else {
+                    warn!(
+                        "kprobe copy_memory: user addr {:#x} not mapped",
+                        vaddr.as_usize()
+                    );
+                    return;
+                };
+                let page_off = vaddr.as_usize() & (PAGE_SIZE_4K - 1);
+                let chunk = core::cmp::min(len - copied, PAGE_SIZE_4K - page_off);
+                let kvaddr = ax_runtime::hal::mem::phys_to_virt(paddr);
+                unsafe {
+                    core::ptr::copy_nonoverlapping(kvaddr.as_ptr(), dst.add(copied), chunk);
                 }
+                copied += chunk;
             }
         } else {
             unsafe {
@@ -70,8 +96,28 @@ impl KprobeAuxiliaryOps for KernelKprobeOps {
         user_pid: Option<i32>,
         action: F,
     ) {
-        if user_pid.is_some() {
-            unimplemented!("user space breakpoint insertion not yet supported")
+        if let Some(pid) = user_pid {
+            // User-space probe (uprobe): patch the target process' text by
+            // writing through the *kernel* direct-map alias of the page's
+            // physical frame. The user PTE keeps its read-only/exec flags
+            // untouched (no per-fire `protect` dance needed — uprobe single-step
+            // is out-of-line, see `alloc_user_exec_memory`). This runs at
+            // arm/disarm time (syscall context), so taking the sleeping aspace
+            // lock is fine. The instruction patch (≤ a few bytes) stays within
+            // the resolved page.
+            let task = crate::task::get_task(pid as _).expect("uprobe: target task gone");
+            let aspace = task.as_thread().proc_data.aspace();
+            let mm = aspace.lock();
+            let vaddr = VirtAddr::from(address);
+            let (paddr, ..) = mm
+                .page_table()
+                .query(vaddr)
+                .expect("uprobe: target address not mapped");
+            let kvaddr = ax_runtime::hal::mem::phys_to_virt(paddr);
+            action(kvaddr.as_mut_ptr());
+            crate::mm::flush_tlb_range(vaddr.align_down_4k(), PAGE_SIZE_4K);
+            ax_runtime::hal::cpu::asm::flush_icache_all();
+            return;
         }
         let addr = VirtAddr::from(address);
         let aligned_addr = addr.align_down_4k();
@@ -134,12 +180,52 @@ impl KprobeAuxiliaryOps for KernelKprobeOps {
             .expect("kprobe: unmap exec memory failed");
     }
 
-    fn alloc_user_exec_memory<F: FnOnce(*mut u8)>(_pid: Option<i32>, _action: F) -> *mut u8 {
-        unimplemented!("user exec memory allocation for uprobes not yet supported")
+    fn alloc_user_exec_memory<F: FnOnce(*mut u8)>(pid: Option<i32>, action: F) -> *mut u8 {
+        // Allocate one anonymous, user-executable page in the target process for
+        // out-of-line single-stepping (the displaced original instruction is
+        // copied here so the planted `int3` can stay armed). `action` writes
+        // that instruction through the kernel alias of the freshly-mapped frame.
+        let pid = pid.expect("uprobe: alloc_user_exec_memory needs a pid");
+        let task = crate::task::get_task(pid as _).expect("uprobe: target task gone");
+        let aspace = task.as_thread().proc_data.aspace();
+        let mut mm = aspace.lock();
+        let range = VirtAddrRange::new(mm.base(), mm.end());
+        let vaddr = mm
+            .find_free_area(mm.base(), PAGE_SIZE_4K, range, PAGE_SIZE_4K)
+            .expect("uprobe: no free user va for exec memory");
+        let backend = crate::mm::Backend::new_alloc(
+            vaddr,
+            ax_runtime::hal::paging::PageSize::Size4K,
+            "uprobe-ols",
+        );
+        mm.map(
+            vaddr,
+            PAGE_SIZE_4K,
+            ax_runtime::hal::paging::MappingFlags::READ
+                | ax_runtime::hal::paging::MappingFlags::EXECUTE
+                | ax_runtime::hal::paging::MappingFlags::USER,
+            true,
+            backend,
+        )
+        .expect("uprobe: map user exec memory failed");
+        let (paddr, ..) = mm
+            .page_table()
+            .query(vaddr)
+            .expect("uprobe: exec page not mapped after populate");
+        let kvaddr = ax_runtime::hal::mem::phys_to_virt(paddr);
+        action(kvaddr.as_mut_ptr());
+        crate::mm::flush_tlb_range(vaddr, PAGE_SIZE_4K);
+        ax_runtime::hal::cpu::asm::flush_icache_all();
+        vaddr.as_mut_ptr()
     }
 
-    fn free_user_exec_memory(_pid: Option<i32>, _ptr: *mut u8) {
-        unimplemented!("user exec memory deallocation for uprobes not yet supported")
+    fn free_user_exec_memory(pid: Option<i32>, ptr: *mut u8) {
+        let pid = pid.expect("uprobe: free_user_exec_memory needs a pid");
+        let task = crate::task::get_task(pid as _).expect("uprobe: target task gone");
+        let aspace = task.as_thread().proc_data.aspace();
+        let mut mm = aspace.lock();
+        mm.unmap(VirtAddr::from(ptr as usize), PAGE_SIZE_4K)
+            .expect("uprobe: unmap user exec memory failed");
     }
 
     fn insert_kretprobe_instance_to_task(instance: kprobe::retprobe::RetprobeInstance) {
@@ -157,8 +243,8 @@ impl KprobeAuxiliaryOps for KernelKprobeOps {
     }
 }
 
-type KprobeManager = kprobe::ProbeManager<KernelRawMutex, KernelKprobeOps>;
-type KprobePointList = ProbePointList<KernelKprobeOps>;
+pub(crate) type KprobeManager = kprobe::ProbeManager<KernelRawMutex, KernelKprobeOps>;
+pub(crate) type KprobePointList = ProbePointList<KernelKprobeOps>;
 
 /// Concrete `kprobe::Kprobe` parameterized on the kernel's `RawMutex` and
 /// auxiliary ops, named to match what the perf module expects.
@@ -219,7 +305,7 @@ pub fn unregister_kretprobe(kretprobe: Arc<KernelKretprobe>) {
     with_manager_and_list(|mgr, list| kprobe_crate_unregister_kretprobe(mgr, list, kretprobe));
 }
 
-fn trapframe_to_ptregs(tf: &ax_runtime::hal::cpu::TrapFrame) -> kprobe::PtRegs {
+pub(crate) fn trapframe_to_ptregs(tf: &ax_runtime::hal::cpu::TrapFrame) -> kprobe::PtRegs {
     #[cfg(target_arch = "x86_64")]
     {
         kprobe::PtRegs {
@@ -348,7 +434,7 @@ fn trapframe_to_ptregs(tf: &ax_runtime::hal::cpu::TrapFrame) -> kprobe::PtRegs {
     }
 }
 
-fn ptregs_write_back(pt: &kprobe::PtRegs, tf: &mut ax_runtime::hal::cpu::TrapFrame) {
+pub(crate) fn ptregs_write_back(pt: &kprobe::PtRegs, tf: &mut ax_runtime::hal::cpu::TrapFrame) {
     #[cfg(target_arch = "x86_64")]
     {
         tf.r15 = pt.r15 as u64;
@@ -459,7 +545,9 @@ pub fn handle_breakpoint(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool {
         ptregs_write_back(&pt_regs, tf);
         return true;
     }
-    false
+    // Not a kernel kprobe — it may be a uprobe `int3` planted in the current
+    // process' user text. The uprobe registry is per-process.
+    crate::uprobe::break_uprobe_handler(tf).is_some()
 }
 
 #[allow(dead_code)]
@@ -542,5 +630,6 @@ pub fn handle_debug(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool {
         ptregs_write_back(&pt_regs, tf);
         return true;
     }
-    false
+    // Fall through to the per-process uprobe single-step (out-of-line) handler.
+    crate::uprobe::debug_uprobe_handler(tf).is_some()
 }

os/StarryOS/kernel/src/lib.rs

@@ -33,3 +33,4 @@ mod task;
 mod time;
 mod tracepoint;
 mod trap;
+mod uprobe;