diff --git a/Cargo.lock b/Cargo.lock index a6aac01077..3ca39e4e45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7913,12 +7913,14 @@ dependencies = [ "ax-alloc", "ax-config", "ax-cpu", + "ax-crate-interface", "ax-display", "ax-dma", "ax-driver", "ax-errno 0.6.0", "ax-feat", "ax-fs-ng", + "ax-hal", "ax-input", "ax-io", "ax-ipi", diff --git a/os/StarryOS/kernel/Cargo.toml b/os/StarryOS/kernel/Cargo.toml index 09921408a4..4634e2376a 100644 --- a/os/StarryOS/kernel/Cargo.toml +++ b/os/StarryOS/kernel/Cargo.toml @@ -47,6 +47,7 @@ ax-feat = { workspace = true, features = [ "multitask", "task-ext", + "tracepoint-hooks", "sched-rr", "rtc", @@ -57,9 +58,11 @@ ax-feat = { workspace = true, features = [ ax-alloc = { workspace = true } ax-config.workspace = true +ax-crate-interface.workspace = true ax-display.workspace = true ax-fs = { version = "0.5.16", path = "../../arceos/modules/axfs-ng", package = "ax-fs-ng" } ax-cpu = { workspace = true, features = ["fp-simd"] } +ax-hal.workspace = true ax-input = { workspace = true, optional = true } ax-lazyinit.workspace = true ax-log.workspace = true diff --git a/os/StarryOS/kernel/src/ebpf/mod.rs b/os/StarryOS/kernel/src/ebpf/mod.rs index 860d8a93fe..57216d903c 100644 --- a/os/StarryOS/kernel/src/ebpf/mod.rs +++ b/os/StarryOS/kernel/src/ebpf/mod.rs @@ -54,7 +54,15 @@ pub static BPF_HELPER_FUN_SET: LazyInit> = LazyIni /// `kbpf-basic`. Must be called before `sys_bpf(BPF_PROG_LOAD)` so loaded /// programs can resolve the helper ids referenced in their instructions. pub fn init_ebpf() { - let set = kbpf_basic::helper::init_helper_functions::(); + let mut set = kbpf_basic::helper::init_helper_functions::(); + // aya emits `bpf_probe_read_kernel` (helper id 113) for reads of kernel + // context memory — e.g. `TracePointContext::read_at`, which a cooked + // tracepoint program uses to pull fields out of its sample buffer. + // kbpf-basic only registers the legacy `bpf_probe_read` (id 4), whose raw + // reader is address-space-agnostic in this VM, so alias 113 onto it. + if let Some(&probe_read) = set.get(&4) { + set.entry(113).or_insert(probe_read); + } BPF_HELPER_FUN_SET.init_once(set); } @@ -86,6 +94,7 @@ fn handle_map_create(attr: &bpf_attr) -> AxResult { fn handle_prog_load(attr: &bpf_attr) -> AxResult { let mut meta = BpfProgMeta::try_from_bpf_attr::(attr).into_ax_result()?; + debug!("bpf prog load meta: {meta:#?}"); let prog = load_prog(&mut meta).into_ax_result()?; // bpf prog fds are close-on-exec in Linux as well; see `handle_map_create`. let fd = add_file_like(Arc::new(prog), true)?; diff --git a/os/StarryOS/kernel/src/file/ion.rs b/os/StarryOS/kernel/src/file/ion.rs index 70e7d3804a..7c804d25aa 100644 --- a/os/StarryOS/kernel/src/file/ion.rs +++ b/os/StarryOS/kernel/src/file/ion.rs @@ -79,7 +79,7 @@ impl FileLike for IonBufferFile { Cow::Borrowed("/dev/ion_buffer") } - fn device_mmap(&self, _offset: u64) -> AxResult { + fn device_mmap(&self, _offset: u64, _length: u64) -> AxResult { Ok(DeviceMmap::Physical(self.phys_range(), None)) } } diff --git a/os/StarryOS/kernel/src/file/mod.rs b/os/StarryOS/kernel/src/file/mod.rs index cf83820caa..4149b2c7e6 100644 --- a/os/StarryOS/kernel/src/file/mod.rs +++ b/os/StarryOS/kernel/src/file/mod.rs @@ -176,7 +176,7 @@ pub trait FileLike: Pollable + DowncastSync { Err(AxError::NoSuchDevice) } - fn device_mmap(&self, _offset: u64) -> AxResult { + fn device_mmap(&self, _offset: u64, _length: u64) -> AxResult { Err(AxError::BadFileDescriptor) } diff --git a/os/StarryOS/kernel/src/kprobe.rs b/os/StarryOS/kernel/src/kprobe.rs index 94e0f83000..eb4c38f14d 100644 --- a/os/StarryOS/kernel/src/kprobe.rs +++ b/os/StarryOS/kernel/src/kprobe.rs @@ -21,6 +21,7 @@ use alloc::sync::Arc; use ax_kspin::RawSpinNoIrq; use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr, VirtAddrRange}; +use ax_runtime::hal::paging::{MappingFlags, PageSize}; use kprobe::{ KprobeAuxiliaryOps, KretprobeBuilder, ProbeBuilder, ProbePointList, register_kprobe as kprobe_crate_register_kprobe, @@ -49,13 +50,36 @@ pub struct KernelKprobeOps; impl KprobeAuxiliaryOps for KernelKprobeOps { fn copy_memory(src: *const u8, dst: *mut u8, len: usize, user_pid: Option) { - if let Some(_pid) = user_pid { - unsafe { - let buf = - core::slice::from_raw_parts_mut(dst as *mut core::mem::MaybeUninit, len); - if let Err(e) = starry_vm::vm_read_slice(src, buf) { - warn!("kprobe copy_memory: vm_read_slice failed: {:?}", e); + if let Some(pid) = user_pid { + // Uprobe arm/disarm reads the target process' original text bytes + // while the per-process kprobe manager spin-lock is held (IRQs + // disabled), so the faultable user-access path (`vm_read_slice`, + // which asserts IRQs enabled) cannot be used. Read through the + // *kernel* direct-map alias of the target page's physical frame + // instead — the same aliasing `set_writeable_for_address` uses to + // write. The text page is already resident (the loader executes the + // probed function before arming). + let task = crate::task::get_task(pid as _).expect("Failed to get task for uprobe"); + let aspace = task.as_thread().proc_data.aspace(); + let mm = aspace.lock(); + let pt = mm.page_table(); + let mut copied = 0; + while copied < len { + let vaddr = VirtAddr::from(src as usize + copied); + let Ok((paddr, ..)) = pt.query(vaddr) else { + warn!( + "kprobe copy_memory: user addr {:#x} not mapped", + vaddr.as_usize() + ); + return; + }; + let page_off = vaddr.as_usize() & (PAGE_SIZE_4K - 1); + let chunk = core::cmp::min(len - copied, PAGE_SIZE_4K - page_off); + let kvaddr = ax_runtime::hal::mem::phys_to_virt(paddr); + unsafe { + core::ptr::copy_nonoverlapping(kvaddr.as_ptr(), dst.add(copied), chunk); } + copied += chunk; } } else { unsafe { @@ -70,8 +94,28 @@ impl KprobeAuxiliaryOps for KernelKprobeOps { user_pid: Option, action: F, ) { - if user_pid.is_some() { - unimplemented!("user space breakpoint insertion not yet supported") + if let Some(pid) = user_pid { + // User-space probe (uprobe): patch the target process' text by + // writing through the *kernel* direct-map alias of the page's + // physical frame. The user PTE keeps its read-only/exec flags + // untouched (no per-fire `protect` dance needed — uprobe single-step + // is out-of-line, see `alloc_user_exec_memory`). This runs at + // arm/disarm time (syscall context), so taking the sleeping aspace + // lock is fine. The instruction patch (≤ a few bytes) stays within + // the resolved page. + let task = crate::task::get_task(pid as _).expect("uprobe: target task gone"); + let aspace = task.as_thread().proc_data.aspace(); + let mm = aspace.lock(); + let vaddr = VirtAddr::from(address); + let (paddr, ..) = mm + .page_table() + .query(vaddr) + .expect("uprobe: target address not mapped"); + let kvaddr = ax_runtime::hal::mem::phys_to_virt(paddr); + action(kvaddr.as_mut_ptr()); + crate::mm::flush_tlb_range(vaddr.align_down_4k(), PAGE_SIZE_4K); + ax_runtime::hal::cpu::asm::flush_icache_all(); + return; } let addr = VirtAddr::from(address); let aligned_addr = addr.align_down_4k(); @@ -89,7 +133,7 @@ impl KprobeAuxiliaryOps for KernelKprobeOps { .protect( aligned_addr, aligned_length, - original_flags | ax_runtime::hal::paging::MappingFlags::WRITE, + original_flags | MappingFlags::WRITE, ) .expect("kprobe: set_writeable: protect failed"); crate::mm::flush_tlb_range(aligned_addr, aligned_length); @@ -117,9 +161,7 @@ impl KprobeAuxiliaryOps for KernelKprobeOps { .map_alloc( vaddr, PAGE_SIZE_4K, - ax_runtime::hal::paging::MappingFlags::READ - | ax_runtime::hal::paging::MappingFlags::WRITE - | ax_runtime::hal::paging::MappingFlags::EXECUTE, + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, true, ) .expect("kprobe: map_alloc for exec memory failed"); @@ -134,12 +176,46 @@ impl KprobeAuxiliaryOps for KernelKprobeOps { .expect("kprobe: unmap exec memory failed"); } - fn alloc_user_exec_memory(_pid: Option, _action: F) -> *mut u8 { - unimplemented!("user exec memory allocation for uprobes not yet supported") + fn alloc_user_exec_memory(pid: Option, action: F) -> *mut u8 { + // Allocate one anonymous, user-executable page in the target process for + // out-of-line single-stepping (the displaced original instruction is + // copied here so the planted `int3` can stay armed). `action` writes + // that instruction through the kernel alias of the freshly-mapped frame. + let pid = pid.expect("uprobe: alloc_user_exec_memory needs a pid"); + let task = crate::task::get_task(pid as _).expect("uprobe: target task gone"); + let aspace = task.as_thread().proc_data.aspace(); + let mut mm = aspace.lock(); + let range = VirtAddrRange::new(mm.base(), mm.end()); + let vaddr = mm + .find_free_area(mm.base(), PAGE_SIZE_4K, range, PAGE_SIZE_4K) + .expect("uprobe: no free user va for exec memory"); + let backend = crate::mm::Backend::new_alloc(vaddr, PageSize::Size4K, "uprobe-ols"); + mm.map( + vaddr, + PAGE_SIZE_4K, + MappingFlags::READ | MappingFlags::EXECUTE | MappingFlags::USER, + true, + backend, + ) + .expect("uprobe: map user exec memory failed"); + let (paddr, ..) = mm + .page_table() + .query(vaddr) + .expect("uprobe: exec page not mapped after populate"); + let kvaddr = ax_runtime::hal::mem::phys_to_virt(paddr); + action(kvaddr.as_mut_ptr()); + crate::mm::flush_tlb_range(vaddr, PAGE_SIZE_4K); + ax_runtime::hal::cpu::asm::flush_icache_all(); + vaddr.as_mut_ptr() } - fn free_user_exec_memory(_pid: Option, _ptr: *mut u8) { - unimplemented!("user exec memory deallocation for uprobes not yet supported") + fn free_user_exec_memory(pid: Option, ptr: *mut u8) { + let pid = pid.expect("uprobe: free_user_exec_memory needs a pid"); + let task = crate::task::get_task(pid as _).expect("uprobe: target task gone"); + let aspace = task.as_thread().proc_data.aspace(); + let mut mm = aspace.lock(); + mm.unmap(VirtAddr::from(ptr as usize), PAGE_SIZE_4K) + .expect("uprobe: unmap user exec memory failed"); } fn insert_kretprobe_instance_to_task(instance: kprobe::retprobe::RetprobeInstance) { @@ -157,8 +233,8 @@ impl KprobeAuxiliaryOps for KernelKprobeOps { } } -type KprobeManager = kprobe::ProbeManager; -type KprobePointList = ProbePointList; +pub(crate) type KprobeManager = kprobe::ProbeManager; +pub(crate) type KprobePointList = ProbePointList; /// Concrete `kprobe::Kprobe` parameterized on the kernel's `RawMutex` and /// auxiliary ops, named to match what the perf module expects. @@ -219,7 +295,7 @@ pub fn unregister_kretprobe(kretprobe: Arc) { with_manager_and_list(|mgr, list| kprobe_crate_unregister_kretprobe(mgr, list, kretprobe)); } -fn trapframe_to_ptregs(tf: &ax_runtime::hal::cpu::TrapFrame) -> kprobe::PtRegs { +pub(crate) fn trapframe_to_ptregs(tf: &ax_runtime::hal::cpu::TrapFrame) -> kprobe::PtRegs { #[cfg(target_arch = "x86_64")] { kprobe::PtRegs { @@ -348,7 +424,7 @@ fn trapframe_to_ptregs(tf: &ax_runtime::hal::cpu::TrapFrame) -> kprobe::PtRegs { } } -fn ptregs_write_back(pt: &kprobe::PtRegs, tf: &mut ax_runtime::hal::cpu::TrapFrame) { +pub(crate) fn ptregs_write_back(pt: &kprobe::PtRegs, tf: &mut ax_runtime::hal::cpu::TrapFrame) { #[cfg(target_arch = "x86_64")] { tf.r15 = pt.r15 as u64; @@ -459,7 +535,9 @@ pub fn handle_breakpoint(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool { ptregs_write_back(&pt_regs, tf); return true; } - false + // Not a kernel kprobe — it may be a uprobe `int3` planted in the current + // process' user text. The uprobe registry is per-process. + crate::uprobe::break_uprobe_handler(tf).is_some() } #[allow(dead_code)] @@ -542,5 +620,6 @@ pub fn handle_debug(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool { ptregs_write_back(&pt_regs, tf); return true; } - false + // Fall through to the per-process uprobe single-step (out-of-line) handler. + crate::uprobe::debug_uprobe_handler(tf).is_some() } diff --git a/os/StarryOS/kernel/src/lib.rs b/os/StarryOS/kernel/src/lib.rs index 96b637fca3..9a0f580510 100644 --- a/os/StarryOS/kernel/src/lib.rs +++ b/os/StarryOS/kernel/src/lib.rs @@ -33,3 +33,4 @@ mod task; mod time; mod tracepoint; mod trap; +mod uprobe; diff --git a/os/StarryOS/kernel/src/mm/access.rs b/os/StarryOS/kernel/src/mm/access.rs index 80d0c67689..ecf510a223 100644 --- a/os/StarryOS/kernel/src/mm/access.rs +++ b/os/StarryOS/kernel/src/mm/access.rs @@ -493,11 +493,21 @@ pub fn write_kernel_text(addr: VirtAddr, data: &[u8]) -> AxResult<()> { let aligned_addr = addr.align_down_4k(); let aligned_length = (addr + data.len()).align_up_4k() - aligned_addr; - let mut guard = ax_mm::kernel_aspace().lock(); - let (_, original_flags, _) = guard.page_table().query(aligned_addr)?; - + // The kernel address-space lock (`SpinNoIrq`) MUST be acquired *inside* the + // `stop_machine` critical section, not before it. `stop_machine` itself + // takes a `SpinNoIrq` (`STOP_MACHINE_LOCK`); acquiring `kernel_aspace` + // first and then dropping it inside the closure produces a non-LIFO nesting + // of two IRQ-saving guards, which crosses their saved IRQ states and leaks + // an IRQ-disabled state out of this function. That stranded state later + // trips the atomic-context guard (e.g. `clear_proc_shm` on process exit + // right after a static-key `disable_key`). Nesting it LIFO here keeps the + // IRQ flag balanced — this mirrors the kprobe `set_writeable_for_address` + // path. crate::stop_machine::stop_machine( move || -> AxResult<()> { + let mut guard = ax_mm::kernel_aspace().lock(); + let (_, original_flags, _) = guard.page_table().query(aligned_addr)?; + guard.protect( aligned_addr, aligned_length, diff --git a/os/StarryOS/kernel/src/perf/bpf.rs b/os/StarryOS/kernel/src/perf/bpf.rs index f0e0e25066..321adc45af 100644 --- a/os/StarryOS/kernel/src/perf/bpf.rs +++ b/os/StarryOS/kernel/src/perf/bpf.rs @@ -1,18 +1,21 @@ //! Software perf event with BPF program attachment + ringbuf output. //! -//! Ported from `Starry-OS/StarryOS:ebpf-kmod` (`kernel/src/perf/bpf.rs`). -//! The user-visible ringbuf is created by `BpfPerfEvent::do_mmap`; tgoskits' -//! `FileLike` does not expose a `custom_mmap()` hook on perf-event fds at -//! the moment (#805 + #673 do not introduce one), so the mmap pathway is -//! reachable only through internal callers for now — a follow-up PR will -//! wire it into `mmap(2)`. The rest of the event lifecycle (enable / -//! disable / set_bpf_prog / write_event) matches the upstream behaviour. - -use alloc::sync::Arc; +//! The user-visible ringbuf is created on the first `mmap(perf_fd, ...)` +//! call: `BpfPerfEventWrapper::device_mmap` allocates `1 + 2^N` physically +//! contiguous 4 K pages (header page + power-of-two-page data ring) and +//! hands the kernel virtual address to `BpfPerfEvent::do_mmap`, which +//! initializes `perf_event_mmap_page` in page 0. `sys_mmap` then maps the +//! same physical range into the caller's address space, so user reads of +//! `data_head` / `data_tail` and kernel writes via `bpf_perf_event_output` +//! share one buffer. + +use alloc::sync::{Arc, Weak}; use core::{any::Any, fmt::Debug}; +use ax_alloc::GlobalPage; use ax_errno::{AxError, AxResult}; -use ax_memory_addr::PhysAddr; +use ax_hal::mem::virt_to_phys; +use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr}; use axpoll::{IoEvents, PollSet, Pollable}; use kbpf_basic::{ linux_bpf::perf_event_sample_format, @@ -28,13 +31,38 @@ use crate::{ }; /// Wraps `kbpf_basic::perf::bpf::BpfPerfEvent` with kernel state: a poll -/// set so readers can wait for new records, and the backing -/// `(PhysAddr, page_count)` produced by `do_mmap` (Some after the user -/// `mmap`s the ringbuf; None before). +/// set so readers can wait for new records, and a weak handle to the +/// backing pages produced by `device_mmap` (Some after the first +/// `mmap(perf_fd)`; None before). +/// +/// Ownership model: the user VMA owns the ringbuf pages via the strong +/// `Arc` threaded into `DeviceMmap::Physical`'s retainer slot; +/// this wrapper keeps only a `Weak`. Consequences: +/// +/// * UAF safety — the pages outlive `close(perf_fd)` (which drops this +/// wrapper) for as long as a mapping is live, because the VMA holds the +/// strong ref. A userspace read after closing the fd never observes +/// freed memory. +/// * Self-cleaning allocation — if a `device_mmap` result is never adopted +/// by a surviving VMA (a non-direct mmap path, a permission/address +/// error, or an `aspace.map` failure), the lone strong ref drops, the +/// frames free, and `is_mapped` flips back to false so the perf fd can +/// be mmap'd again instead of being wedged in `ResourceBusy`. After a +/// normal `munmap` the same thing happens, matching Linux's allowance to +/// re-`mmap` a perf fd. +/// +/// `inner` holds a raw pointer into the page buffer; `RingPage` has no +/// destructor and is never dereferenced once the pages are gone (every +/// access through `inner` is gated on [`Self::is_mapped`]), so a dangling +/// pointer left after the pages free is harmless. pub struct BpfPerfEventWrapper { inner: BpfPerfEvent, poll_ready: PollSet, - phys_addr: Option<(PhysAddr, usize)>, + /// Weak handle to the contiguous pages backing the ringbuf. The strong + /// ref(s) live in the user VMA(s); `strong_count() > 0` means a live + /// mapping still exists. See the type-level docs for the ownership + /// rationale. + pages: Option>, } impl BpfPerfEventWrapper { @@ -43,16 +71,25 @@ impl BpfPerfEventWrapper { Self { inner, poll_ready: PollSet::new(), - phys_addr: None, + pages: None, } } - /// Write a record into the ringbuf and wake any readers. Pre-mmap - /// calls are accepted as no-ops (matching the source behaviour). + /// Whether a live user mapping of the ringbuf currently exists. The + /// wrapper only holds a `Weak` to the backing pages, so this is true + /// exactly while some VMA still pins them; once every mapping is gone + /// (munmap / exit) — or an in-progress mmap was abandoned before a VMA + /// adopted the pages — the strong refs drop and this returns false. + fn is_mapped(&self) -> bool { + self.pages.as_ref().is_some_and(|w| w.strong_count() > 0) + } + + /// Write a record into the ringbuf and wake any readers. Calls before a + /// mapping exists (or after it is gone) are accepted as no-ops: the + /// `kbpf_basic::RingPage` pointer is either still `empty()` or now + /// dangling, so dereferencing it would be UB. pub fn write_event(&mut self, data: &[u8]) -> AxResult<()> { - if self.phys_addr.is_none() { - // Ringbuf not yet mapped by userland; drop the sample silently - // — Linux behavior on EINVAL would alarm libbpf-style readers. + if !self.is_mapped() { return Ok(()); } self.inner.write_event(data).into_ax_result()?; @@ -83,15 +120,44 @@ impl PerfEventOps for BpfPerfEventWrapper { fn as_any_mut(&mut self) -> &mut dyn Any { self } -} -impl Drop for BpfPerfEventWrapper { - fn drop(&mut self) { - // The mmap'd ringbuf pages, if any, were allocated via the global - // page allocator in the (not yet wired) mmap path; once that path - // lands, this drop will need to call `frame_dealloc` for each. For - // now the field stays `None`, so this is effectively a no-op. - let _ = self.phys_addr.take(); + fn device_mmap(&mut self, len: usize) -> AxResult<(PhysAddr, Arc)> { + if self.is_mapped() { + // Linux allows only one live mmap per perf event fd; a second + // mapping while the first is alive would orphan it. A stale + // `Weak` from an abandoned or munmap'd previous attempt does not + // count (its pages are already freed), so the fd stays mmap-able. + return Err(AxError::ResourceBusy); + } + // libbpf requires `(1 + 2^N) * PAGE_SIZE` so the data region is a + // power of two pages; `RingPage::init` enforces ≥ 2 pages total and + // 4 K alignment. Reject anything that would trip those asserts. + if len == 0 || !len.is_multiple_of(PAGE_SIZE_4K) { + return Err(AxError::InvalidInput); + } + let num_pages = len / PAGE_SIZE_4K; + if num_pages < 2 || !(num_pages - 1).is_power_of_two() { + return Err(AxError::InvalidInput); + } + let mut pages = GlobalPage::alloc_contiguous(num_pages, PAGE_SIZE_4K)?; + pages.zero(); + let kvirt = pages.start_vaddr(); + let paddr = virt_to_phys(kvirt); + self.inner + .do_mmap(kvirt.as_usize(), len, 0) + .map_err(|_| AxError::InvalidInput)?; + let pages = Arc::new(pages); + // Keep only a `Weak`; hand the sole strong ref to the caller, which + // threads it into `DeviceMmap::Physical`'s retainer so the user VMA + // pins these frames until `munmap`/exit even if the perf fd (and this + // wrapper) is closed first. Because the wrapper does not retain a + // strong ref, an mmap that is abandoned or fails before a VMA adopts + // the anchor simply frees the pages and leaves the fd mmap-able again + // (see the type-level docs). Without the anchor the pages would free + // under a live mapping. + self.pages = Some(Arc::downgrade(&pages)); + let anchor: Arc = pages; + Ok((paddr, anchor)) } } diff --git a/os/StarryOS/kernel/src/perf/kprobe.rs b/os/StarryOS/kernel/src/perf/kprobe.rs index 6060467490..352e17a61b 100644 --- a/os/StarryOS/kernel/src/perf/kprobe.rs +++ b/os/StarryOS/kernel/src/perf/kprobe.rs @@ -23,17 +23,17 @@ use crate::{ register_kretprobe, unregister_kprobe, unregister_kretprobe, }, perf::{PerfEventOps, bpf::OwnedEbpfVm}, + uprobe::{KernelUprobe, unregister_uprobe}, }; -/// One of {kprobe, kretprobe}. Uprobe is *not* exposed through -/// `ProbeTy` because tgoskits does not yet provide the uprobe manager -/// infrastructure the source assumes (`ProcessData::uprobe_manager`, -/// `ProcessData::uprobe_point_list`). The uprobe perf event variant -/// returns `Unsupported` until that infrastructure lands. +/// One of {kprobe, kretprobe, uprobe}. Kprobe/kretprobe live in the global +/// kernel-text manager; uprobe lives in the firing process' per-process manager +/// (`ProcessData::uprobe_manager`), but exposes the same probe API. #[derive(Debug)] pub enum ProbeTy { Kprobe(Arc), Kretprobe(Arc), + Uprobe(Arc), } /// Per-fd perf event wrapping a kprobe/kretprobe registration. @@ -61,11 +61,13 @@ impl Drop for ProbePerfEvent { match self.probe { ProbeTy::Kprobe(ref k) => k.unregister_event_callback(*cid), ProbeTy::Kretprobe(ref k) => k.unregister_event_callback(*cid), + ProbeTy::Uprobe(ref u) => u.unregister_event_callback(*cid), } } match self.probe { ProbeTy::Kprobe(ref k) => unregister_kprobe(k.clone()), ProbeTy::Kretprobe(ref k) => unregister_kretprobe(k.clone()), + ProbeTy::Uprobe(ref u) => unregister_uprobe(u.clone()), } } } @@ -87,6 +89,7 @@ impl PerfEventOps for ProbePerfEvent { match self.probe { ProbeTy::Kprobe(ref k) => k.enable(), ProbeTy::Kretprobe(ref k) => k.kprobe().enable(), + ProbeTy::Uprobe(ref u) => u.enable(), } Ok(()) } @@ -95,6 +98,7 @@ impl PerfEventOps for ProbePerfEvent { match self.probe { ProbeTy::Kprobe(ref k) => k.disable(), ProbeTy::Kretprobe(ref k) => k.kprobe().disable(), + ProbeTy::Uprobe(ref u) => u.disable(), } Ok(()) } @@ -120,6 +124,7 @@ impl PerfEventOps for ProbePerfEvent { match self.probe { ProbeTy::Kprobe(ref k) => k.register_event_callback(id, callback), ProbeTy::Kretprobe(ref k) => k.register_event_callback(id, callback), + ProbeTy::Uprobe(ref u) => u.register_event_callback(id, callback), } self.callback_list.push(id); Ok(()) diff --git a/os/StarryOS/kernel/src/perf/mod.rs b/os/StarryOS/kernel/src/perf/mod.rs index 8fdd028295..4c186c1c1a 100644 --- a/os/StarryOS/kernel/src/perf/mod.rs +++ b/os/StarryOS/kernel/src/perf/mod.rs @@ -1,13 +1,10 @@ //! `perf_event_open(2)` runtime: dispatcher across kprobe / tracepoint / //! software-bpf / uprobe perf event types, the file-like `PerfEvent` //! wrapper, and the ringbuf output path used by the `bpf_perf_event_output` -//! helper. -//! -//! Ported from `Starry-OS/StarryOS:ebpf-kmod` (`kernel/src/perf/`). Adapted -//! to tgoskits' `ax_*` package naming and the in-tree FileLike trait (which -//! does not expose the source's `custom_mmap()` hook — ringbuf mmap is -//! handled inside the relevant `set_bpf_prog`/`enable` paths rather than -//! through the fd layer for now; see TODO in `perf/bpf.rs`). +//! helper. The `mmap(perf_fd, ...)` path is wired through +//! `FileLike::device_mmap` → `PerfEventOps::device_mmap`, which allocates +//! the backing pages and asks `kbpf_basic` to initialize the +//! `perf_event_mmap_page` header. pub mod bpf; pub mod kprobe; @@ -22,6 +19,7 @@ use ax_errno::{AxError, AxResult}; use ax_io::Read; use ax_kspin::{SpinNoPreempt, SpinNoPreemptGuard}; use ax_lazyinit::LazyInit; +use ax_memory_addr::{PhysAddr, PhysAddrRange}; use axpoll::Pollable; pub use bpf::BpfPerfEventWrapper; use hashbrown::HashMap; @@ -34,6 +32,7 @@ use crate::{ ebpf::{error::BpfResultExt, transform::EbpfKernelAuxiliary}, file::{FileLike, Kstat, add_file_like, get_file_like}, mm::VmBytes, + pseudofs::DeviceMmap, }; /// Behaviour every perf event implements. Each variant in the dispatcher @@ -55,6 +54,18 @@ pub trait PerfEventOps: Pollable + Send + Sync + Debug { fn set_bpf_prog(&mut self, _bpf_prog: Arc) -> AxResult<()> { Err(AxError::Unsupported) } + + /// Allocate the user-visible ringbuf and return its physical start + /// address (length is the user-supplied mmap length, page-aligned) + /// together with a retainer that owns the backing pages. The caller + /// threads the retainer into `DeviceMmap::Physical(.., Some(anchor))` + /// so the pages stay live for as long as the user mapping exists, even + /// after `close(perf_fd)`. Only `bpf::BpfPerfEventWrapper` overrides + /// this; the other variants (kprobe/tracepoint/raw-tp/uprobe wrappers) + /// reject `mmap(perf_fd)`. + fn device_mmap(&mut self, _len: usize) -> AxResult<(PhysAddr, Arc)> { + Err(AxError::Unsupported) + } } /// File-like handle returned by `perf_event_open(2)`. Locks a @@ -127,6 +138,24 @@ impl FileLike for PerfEvent { } Ok(0) } + + fn device_mmap(&self, offset: u64, length: u64) -> AxResult { + // libbpf calls mmap with offset == 0; non-zero offsets address into + // the ringbuf, which has no meaningful sub-region exposed as a fd + // offset (data_offset lives inside the header page). + if offset != 0 { + return Err(AxError::InvalidInput); + } + let len = length as usize; + let (paddr, anchor) = self.event.lock().device_mmap(len)?; + // Anchor the ringbuf pages to the VMA: the retainer keeps them alive + // until `munmap`/exit, so closing the perf fd can't free memory the + // user address space still maps. See `BpfPerfEventWrapper::pages`. + Ok(DeviceMmap::Physical( + PhysAddrRange::from_start_size(paddr, len), + Some(anchor), + )) + } } /// `perf_event_open(2)` syscall entry. Copies the user `perf_event_attr` in diff --git a/os/StarryOS/kernel/src/perf/raw_tracepoint.rs b/os/StarryOS/kernel/src/perf/raw_tracepoint.rs index 06dd9b9912..a60cb448bb 100644 --- a/os/StarryOS/kernel/src/perf/raw_tracepoint.rs +++ b/os/StarryOS/kernel/src/perf/raw_tracepoint.rs @@ -86,8 +86,12 @@ impl RawTracepointPerfEvent { }); let func: RawTpCallback = Box::new(|args: &[u64], data: &(dyn Any + Send + Sync)| { + // `RawTraceEventFunc` keeps the payload as `Box` + // and hands the closure `&self.data`, so the concrete type observed + // here is the *box*, not `Ctx`. Downcast through the box first. let ctx = data - .downcast_ref::() + .downcast_ref::>() + .and_then(|boxed| boxed.downcast_ref::()) .expect("raw_tracepoint Ctx mismatch"); // SAFETY: raw tracepoint hands us the raw `&[u64]` arg // slice on the tracing fast path; the slice lives for the diff --git a/os/StarryOS/kernel/src/perf/tracepoint.rs b/os/StarryOS/kernel/src/perf/tracepoint.rs index 5ad9813c81..0a06d84fd5 100644 --- a/os/StarryOS/kernel/src/perf/tracepoint.rs +++ b/os/StarryOS/kernel/src/perf/tracepoint.rs @@ -87,7 +87,14 @@ impl PerfEventOps for TracepointPerfEvent { }); let func: TpCallback = Box::new(|entry: &[u8], data: &(dyn Any + Send + Sync)| { - let ctx = data.downcast_ref::().expect("tracepoint Ctx mismatch"); + // `TraceEventFunc` keeps the payload as `Box` + // and hands the closure `&self.data`, so the concrete type observed + // here is the *box*, not `Ctx` (same as the raw-tracepoint path in + // `raw_tracepoint.rs`). Downcast through the box first. + let ctx = data + .downcast_ref::>() + .and_then(|boxed| boxed.downcast_ref::()) + .expect("tracepoint Ctx mismatch"); // BPF programs expect a mutable context slice; the // tracepoint hands us a `&[u8]` carved out of its // per-cpu sample buffer, which is single-writer at that diff --git a/os/StarryOS/kernel/src/perf/uprobe.rs b/os/StarryOS/kernel/src/perf/uprobe.rs index 29cd8f13d6..84374b2775 100644 --- a/os/StarryOS/kernel/src/perf/uprobe.rs +++ b/os/StarryOS/kernel/src/perf/uprobe.rs @@ -1,24 +1,92 @@ -//! Uprobe perf event. Tracks the kernel-side -//! [`crate::uprobe::KernelUprobe`] registration; the source resolves the -//! ELF base address by walking the target process' VMAs, which requires -//! the uprobe_manager / uprobe_point_list fields on `ProcessData` and an -//! `AddrSpace::memoryset()` accessor that tgoskits has not introduced yet -//! (post-#805 / #673 baselines lack both). +//! Uprobe perf event. //! -//! Until that infrastructure lands (planned for a follow-up that extends -//! `ProcessData`), opening a uprobe perf event surfaces -//! [`AxError::Unsupported`] rather than panicking the kernel. The -//! placeholder still threads the `PerfProbeArgs` through so the upper -//! layers compile end-to-end. +//! `perf_event_open(PERF_TYPE_UPROBE)` carries the target ELF path (`name`), an +//! in-file `offset`, and the target `pid`. We resolve the offset to a live user +//! virtual address by finding the VMA in the target process that is backed by +//! that ELF (`Backend::file_info().path`), then register a uprobe on +//! `vma.start() + offset` in that process' per-process manager. +//! +//! Out-of-line single-step, breakpoint insertion and the eBPF callback are +//! handled by the `kprobe` crate via the user-mode `KprobeAuxiliaryOps` paths +//! in [`crate::kprobe`]; this module only does the address resolution and +//! per-process registration. use ax_errno::{AxError, AxResult}; -use kbpf_basic::perf::PerfProbeArgs; +use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig}; +use kprobe::ProbeBuilder; + +use super::kprobe::{ProbePerfEvent, ProbeTy}; +use crate::{ + kprobe::KprobeAuxiliary, + task::{AsThread, get_task}, +}; + +/// Resolve the target ELF's mapped base in the target process and build a +/// uprobe `ProbeBuilder` for `base + offset`. +fn perf_probe_arg_to_uprobe_builder( + args: &PerfProbeArgs, +) -> AxResult> { + let elf = &args.name; + let offset = args.offset as usize; + let pid = args.pid; + + if pid < 0 { + // pid == -1 means "all processes" (e.g. a shared-library uprobe). That + // needs a global file→address registry we do not maintain. + warn!("uprobe: pid == -1 (all-process / shared-lib uprobe) is unsupported"); + return Err(AxError::Unsupported); + } + + let task = get_task(pid as _)?; + let aspace = task.as_thread().proc_data.aspace(); + let mm = aspace.lock(); -use super::kprobe::ProbePerfEvent; + let mut virt_base = None; + for area in mm.areas() { + if let Ok(info) = area.backend().file_info() + && &info.path == elf + { + virt_base = Some(area.start()); + break; + } + } + drop(mm); + + let Some(virt_base) = virt_base else { + warn!("uprobe: ELF {elf} is not mapped in pid {pid}"); + return Err(AxError::NotFound); + }; + + let virt_addr = virt_base.as_usize() + offset; + debug!( + "uprobe: pid {pid} ELF {elf} base {:#x} + offset {:#x} = {virt_addr:#x}", + virt_base.as_usize(), + offset + ); + + Ok(ProbeBuilder::new() + .with_symbol(elf.clone()) + .with_symbol_addr(virt_addr) + .with_offset(0) + .with_user_mode(pid)) +} -/// Build a uprobe perf event from `perf_event_open` args. Always returns -/// `Unsupported` for now; see module docs. -pub fn perf_event_open_uprobe(_args: PerfProbeArgs) -> AxResult { - warn!("perf_event_open(PERF_TYPE_UPROBE) is not yet supported on tgoskits"); - Err(AxError::Unsupported) +/// Build a uprobe perf event from `perf_event_open` args. +pub fn perf_event_open_uprobe(args: PerfProbeArgs) -> AxResult { + let probe = match args.config { + PerfProbeConfig::Raw(0) => { + let builder = perf_probe_arg_to_uprobe_builder(&args)?; + ProbeTy::Uprobe(crate::uprobe::register_uprobe(builder)) + } + PerfProbeConfig::Raw(1) => { + // uretprobe — not implemented for user space yet. + warn!("uprobe: uretprobe (config=1) is not yet supported"); + return Err(AxError::Unsupported); + } + other => { + warn!("uprobe: unsupported perf probe config {other:?}"); + return Err(AxError::Unsupported); + } + }; + Ok(ProbePerfEvent::new(args, probe)) } diff --git a/os/StarryOS/kernel/src/pseudofs/dev/card1.rs b/os/StarryOS/kernel/src/pseudofs/dev/card1.rs index da7b1181b8..a32facb7c0 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/card1.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/card1.rs @@ -210,7 +210,7 @@ impl FileLike for ExportedGemBuffer { "anon_inode:[rknpu-gem]".into() } - fn device_mmap(&self, _offset: u64) -> AxResult { + fn device_mmap(&self, _offset: u64, _length: u64) -> AxResult { Ok(DeviceMmap::Physical(self.range, None)) } } @@ -547,7 +547,7 @@ mod tests { let exported = ExportedGemBuffer::new(range); assert!( - matches!(exported.device_mmap(0).unwrap(), DeviceMmap::Physical(actual, None) if actual == range) + matches!(exported.device_mmap(0, 0).unwrap(), DeviceMmap::Physical(actual, None) if actual == range) ); } } diff --git a/os/StarryOS/kernel/src/pseudofs/sysfs.rs b/os/StarryOS/kernel/src/pseudofs/sysfs.rs index 7f3b5750de..cbb9b84c5f 100644 --- a/os/StarryOS/kernel/src/pseudofs/sysfs.rs +++ b/os/StarryOS/kernel/src/pseudofs/sysfs.rs @@ -304,9 +304,9 @@ struct BusDir { impl SimpleDirOps for BusDir { fn child_names<'a>(&'a self) -> Box> + 'a> { #[cfg(feature = "plat-dyn")] - let names: &'static [&'static str] = &["platform", "usb"]; + let names: &'static [&'static str] = &["platform", "usb", "event_source"]; #[cfg(not(feature = "plat-dyn"))] - let names: &'static [&'static str] = &["platform"]; + let names: &'static [&'static str] = &["platform", "event_source"]; Box::new(names.iter().copied().map(Cow::Borrowed)) } @@ -316,6 +316,9 @@ impl SimpleDirOps for BusDir { "platform" => SimpleDir::new_maker(fs.clone(), Arc::new(PlatformBusClassDir)), #[cfg(feature = "plat-dyn")] "usb" => SimpleDir::new_maker(fs.clone(), Arc::new(DirMapping::new())), + "event_source" => { + SimpleDir::new_maker(fs.clone(), Arc::new(EventSourceBusDir { fs: fs.clone() })) + } _ => return Err(VfsError::NotFound), })) } @@ -333,6 +336,123 @@ impl SimpleDirOps for PlatformBusClassDir { } } +// /sys/bus/event_source/devices//type — aya reads this to learn the +// dynamic perf_event_type for each event source (kprobe / uprobe / tracepoint). +// Values match `kbpf_basic::perf::PerfTypeId` so the user-supplied number +// dispatches cleanly in `perf_event_open`. +const PERF_EVENT_SOURCES: &[(&str, u32)] = &[ + ("kprobe", 6), // PerfTypeId::PERF_TYPE_KPROBE + ("uprobe", 7), // PerfTypeId::PERF_TYPE_UPROBE + ("tracepoint", 2), // PERF_TYPE_TRACEPOINT +]; + +struct EventSourceBusDir { + fs: Arc, +} + +impl SimpleDirOps for EventSourceBusDir { + fn child_names<'a>(&'a self) -> Box> + 'a> { + Box::new(["devices"].into_iter().map(Cow::Borrowed)) + } + + fn lookup_child(&self, name: &str) -> VfsResult { + let fs = self.fs.clone(); + Ok(NodeOpsMux::Dir(match name { + "devices" => SimpleDir::new_maker( + fs.clone(), + Arc::new(EventSourceDevicesDir { fs: fs.clone() }), + ), + _ => return Err(VfsError::NotFound), + })) + } +} + +struct EventSourceDevicesDir { + fs: Arc, +} + +impl SimpleDirOps for EventSourceDevicesDir { + fn child_names<'a>(&'a self) -> Box> + 'a> { + Box::new(PERF_EVENT_SOURCES.iter().map(|(n, _)| Cow::Borrowed(*n))) + } + + fn lookup_child(&self, name: &str) -> VfsResult { + let fs = self.fs.clone(); + let ty = PERF_EVENT_SOURCES + .iter() + .find(|(n, _)| *n == name) + .map(|(_, t)| *t) + .ok_or(VfsError::NotFound)?; + Ok(NodeOpsMux::Dir(SimpleDir::new_maker( + fs.clone(), + Arc::new(EventSourceDeviceDir { fs: fs.clone(), ty }), + ))) + } +} + +struct EventSourceDeviceDir { + fs: Arc, + ty: u32, +} + +impl EventSourceDeviceDir { + /// kprobe (6) / uprobe (7) PMUs support a return-probe variant selected via + /// the `retprobe` config bit; tracepoint (2) does not. + fn supports_retprobe(&self) -> bool { + self.ty == 6 || self.ty == 7 + } +} + +impl SimpleDirOps for EventSourceDeviceDir { + fn child_names<'a>(&'a self) -> Box> + 'a> { + if self.supports_retprobe() { + Box::new(["type", "format"].into_iter().map(Cow::Borrowed)) + } else { + Box::new(["type"].into_iter().map(Cow::Borrowed)) + } + } + + fn lookup_child(&self, name: &str) -> VfsResult { + let fs = self.fs.clone(); + match name { + "type" => { + let body = format!("{}\n", self.ty); + Ok(SimpleFile::new_regular(fs, move || Ok(body.clone())).into()) + } + // `/sys/bus/event_source/devices/probe/format/` — aya reads + // `format/retprobe` to learn which `config` bit selects the + // return-probe variant before `perf_event_open` for a kretprobe / + // uretprobe. + "format" if self.supports_retprobe() => Ok(NodeOpsMux::Dir(SimpleDir::new_maker( + fs.clone(), + Arc::new(EventSourceFormatDir { fs }), + ))), + _ => Err(VfsError::NotFound), + } + } +} + +struct EventSourceFormatDir { + fs: Arc, +} + +impl SimpleDirOps for EventSourceFormatDir { + fn child_names<'a>(&'a self) -> Box> + 'a> { + Box::new(["retprobe"].into_iter().map(Cow::Borrowed)) + } + + fn lookup_child(&self, name: &str) -> VfsResult { + let fs = self.fs.clone(); + match name { + // `config:0` = the retprobe flag lives in bit 0 of `config`, which + // is exactly what `perf_event_open_kprobe` decodes (config 1 = + // kretprobe). Matches the format string the real kernel exposes. + "retprobe" => Ok(SimpleFile::new_regular(fs, || Ok("config:0\n".to_owned())).into()), + _ => Err(VfsError::NotFound), + } + } +} + // ======================================================================== // /sys/devices // ======================================================================== diff --git a/os/StarryOS/kernel/src/syscall/mm/mmap.rs b/os/StarryOS/kernel/src/syscall/mm/mmap.rs index 3c576258fe..ec40b1ad22 100644 --- a/os/StarryOS/kernel/src/syscall/mm/mmap.rs +++ b/os/StarryOS/kernel/src/syscall/mm/mmap.rs @@ -176,7 +176,18 @@ pub fn sys_mmap( } else { Some(get_file_like(fd)?) }; - let mut device_mmap_top = file.as_ref().map(|fl| fl.device_mmap(offset as u64)); + // Only probe `device_mmap` for MAP_SHARED. MAP_PRIVATE always maps + // through the file_mmap/CoW path below and never consumes this result, so + // calling it would be wasted work — and for fds whose `device_mmap` has + // side effects (e.g. a perf-event ringbuf allocation) it would leave the + // fd in a half-initialized state that rejects the later real MAP_SHARED + // mapping. Probe lazily here, then commit it in the MAP_SHARED arm. + let mut device_mmap_top = if matches!(map_type, MmapFlags::SHARED) { + file.as_ref() + .map(|fl| fl.device_mmap(offset as u64, length as u64)) + } else { + None + }; // Validate file_mmap permissions and memfd seals before any destructive // MAP_FIXED unmap (Linux `do_mmap` ordering; avoids tearing down the old diff --git a/os/StarryOS/kernel/src/syscall/mod.rs b/os/StarryOS/kernel/src/syscall/mod.rs index cd408a6072..96965b8eaf 100644 --- a/os/StarryOS/kernel/src/syscall/mod.rs +++ b/os/StarryOS/kernel/src/syscall/mod.rs @@ -25,9 +25,23 @@ pub fn syscall_allows_signal_restart(sysno: usize) -> bool { !matches!(Sysno::new(sysno), Some(Sysno::msgsnd | Sysno::msgrcv)) } +// `#[inline(never)]` keeps `sysno` reachable as a real call target so a kprobe +// planted at its symbol actually fires; its first-argument register also holds +// the raw syscall id, letting a `profile`-style eBPF demo read the syscall +// number directly off the probed register. In release builds LLVM would +// otherwise inline it into `handle_syscall` and the planted `int3` would land +// on a copy that never executes, so the probe would never trigger. +#[inline(never)] +pub fn sysno(id: usize) -> Option { + let Some(sysno) = Sysno::new(id) else { + warn!("Invalid syscall number: {}", id); + return None; + }; + Some(sysno) +} + pub fn handle_syscall(uctx: &mut UserContext) { - let Some(sysno) = Sysno::new(uctx.sysno()) else { - warn!("Invalid syscall number: {}", uctx.sysno()); + let Some(sysno) = sysno(uctx.sysno()) else { uctx.set_retval(-LinuxError::ENOSYS.code() as _); return; }; diff --git a/os/StarryOS/kernel/src/syscall/task/clone.rs b/os/StarryOS/kernel/src/syscall/task/clone.rs index 1be7b49157..38db20111a 100644 --- a/os/StarryOS/kernel/src/syscall/task/clone.rs +++ b/os/StarryOS/kernel/src/syscall/task/clone.rs @@ -80,6 +80,34 @@ bitflags! { } } +// The `sched:sched_process_fork` tracepoint is defined here, next to its sole +// emission site in `CloneArgs::do_clone` (which all of clone/clone3/fork/vfork +// funnel through), so the event schema and the fast-path call stay together. +// Registration into the global `.tracepoint` section is by link section, so +// the definition's module location is immaterial to discovery. +ktracepoint::define_event_trace!( + sched_process_fork, + TP_kops(crate::tracepoint::KernelTraceAux), + TP_system(sched), + TP_PROTO(parent_tid: u64, child_tid: u64), + TP_STRUCT__entry { + parent_tid: u64, + child_tid: u64, + }, + TP_fast_assign { + parent_tid: parent_tid, + child_tid: child_tid, + }, + TP_ident(__entry), + TP_printk({ + alloc::format!( + "parent_tid={} child_tid={}", + __entry.parent_tid, + __entry.child_tid, + ) + }) +); + /// Unified arguments for clone/clone3/fork/vfork. #[derive(Debug, Clone, Copy, Default)] pub struct CloneArgs { @@ -367,6 +395,10 @@ impl CloneArgs { ); } + // Fire before any potential vfork-wait so observers see the fork edge + // even when the parent blocks below. + trace_sched_process_fork(curr.id().as_u64(), tid as u64); + // Block the parent until the child exec's or exits. if needs_vfork_block { new_proc_data.wait_vfork_done(); diff --git a/os/StarryOS/kernel/src/syscall/task/thread.rs b/os/StarryOS/kernel/src/syscall/task/thread.rs index 419b9fc7a8..fd1af79d3a 100644 --- a/os/StarryOS/kernel/src/syscall/task/thread.rs +++ b/os/StarryOS/kernel/src/syscall/task/thread.rs @@ -3,6 +3,7 @@ use ax_task::current; use crate::task::AsThread; +#[inline(never)] pub fn sys_getpid() -> AxResult { let curr = current(); let thr = curr.as_thread(); diff --git a/os/StarryOS/kernel/src/task/mod.rs b/os/StarryOS/kernel/src/task/mod.rs index 7b354cdea5..9939331990 100644 --- a/os/StarryOS/kernel/src/task/mod.rs +++ b/os/StarryOS/kernel/src/task/mod.rs @@ -511,6 +511,16 @@ pub struct ProcessData { /// The virtual memory address space. // TODO: scopify aspace: SpinNoIrq>>, + /// Per-process uprobe manager. Uprobes plant an `int3` in *this* process' + /// user text, so (unlike the global kprobe manager) the registry is + /// per-address-space. A *sleeping* mutex, because arming/disarming + /// manipulates the user address space (page-table query, faulting reads, + /// mapping the out-of-line single-step page) which requires sleeping locks; + /// the exception-context breakpoint/debug handlers acquire it with + /// `try_lock()` (a single CAS, safe in atomic context) instead. + pub uprobe_manager: Mutex, + /// Per-process uprobe point list, paired with [`Self::uprobe_manager`]. + pub uprobe_point_list: Mutex, /// The resource scope pub scope: RwLock, /// The namespace proxy — aggregates all namespace types for this process. @@ -679,6 +689,8 @@ impl ProcessData { cmdline: RwLock::new(image.cmdline), auxv: RwLock::new(image.auxv), aspace: SpinNoIrq::new(aspace), + uprobe_manager: Mutex::new(crate::kprobe::KprobeManager::default()), + uprobe_point_list: Mutex::new(crate::kprobe::KprobePointList::new()), scope: RwLock::new(Scope::new()), heap_top: AtomicUsize::new(crate::config::USER_HEAP_BASE), diff --git a/os/StarryOS/kernel/src/task/ops.rs b/os/StarryOS/kernel/src/task/ops.rs index 9d57885e22..b221ad6ae9 100644 --- a/os/StarryOS/kernel/src/task/ops.rs +++ b/os/StarryOS/kernel/src/task/ops.rs @@ -463,12 +463,41 @@ pub fn exit_robust_list(thr: &Thread, head: *const RobustListHead) -> AxResult<( Ok(()) } +// The `sched:sched_process_exit` tracepoint is defined here, next to its sole +// emission site in `do_exit`, so the event schema and the fast-path call stay +// together. Registration into the global `.tracepoint` section is by link +// section, so the definition's module location is immaterial to discovery. +ktracepoint::define_event_trace!( + sched_process_exit, + TP_kops(crate::tracepoint::KernelTraceAux), + TP_system(sched), + TP_PROTO(tid: u64, exit_code: i32), + TP_STRUCT__entry { + tid: u64, + exit_code: i32, + }, + TP_fast_assign { + tid: tid, + exit_code: exit_code, + }, + TP_ident(__entry), + TP_printk({ + alloc::format!( + "tid={} exit_code={}", + __entry.tid, + __entry.exit_code, + ) + }) +); + pub fn do_exit(exit_code: i32, group_exit: bool) { let curr = current(); let thr = curr.as_thread(); info!("{} exit with code: {}", curr.id_name(), exit_code); + trace_sched_process_exit(curr.id().as_u64(), exit_code); + // Robust futex ownership must be released before clone-child-tid wakes a // pthread joiner; otherwise userspace can observe thread exit before the // OWNER_DIED handoff has been written. diff --git a/os/StarryOS/kernel/src/task/user.rs b/os/StarryOS/kernel/src/task/user.rs index 423ce48ad4..1ead9bd3e7 100644 --- a/os/StarryOS/kernel/src/task/user.rs +++ b/os/StarryOS/kernel/src/task/user.rs @@ -98,6 +98,31 @@ pub fn new_user_task(name: &str, mut uctx: UserContext, set_child_tid: usize) -> #[allow(unused_labels)] ReturnReason::Exception(exc_info) => 'exc: { let kind = exc_info.kind(); + // A uprobe plants an `int3` in user text (delivered as a + // #BP / Breakpoint exception) and completes its + // out-of-line single-step via a #DB / Debug exception. + // Route both to this process' uprobe manager before any + // ptrace / signal handling: if a uprobe owns the + // faulting address it fixes up `uctx` (sets the + // out-of-line PC + single-step, or restores PC after the + // step) and we resume directly. If not, fall through. + match kind { + ExceptionKind::Breakpoint + if crate::uprobe::break_uprobe_handler(&mut uctx).is_some() => + { + break 'exc; + } + // x86_64 completes the out-of-line single-step via a + // #DB; other arches handle stepping inside the + // breakpoint path, so the debug hook is x86_64-only. + #[cfg(target_arch = "x86_64")] + ExceptionKind::Debug + if crate::uprobe::debug_uprobe_handler(&mut uctx).is_some() => + { + break 'exc; + } + _ => {} + } if matches!(kind, ExceptionKind::Breakpoint) && (thr.proc_data.is_ptrace_traceme() || thr.proc_data.is_ptrace_attached()) diff --git a/os/StarryOS/kernel/src/tracepoint/mod.rs b/os/StarryOS/kernel/src/tracepoint/mod.rs index 3741b796dd..0619abb957 100644 --- a/os/StarryOS/kernel/src/tracepoint/mod.rs +++ b/os/StarryOS/kernel/src/tracepoint/mod.rs @@ -1,5 +1,6 @@ //! See Linux Documentation for details: mod control; +mod sched; mod trace; mod trace_pipe; @@ -7,6 +8,7 @@ use alloc::{collections::BTreeMap, string::ToString, sync::Arc, vec::Vec}; use core::{num::NonZero, ops::Deref}; use ax_errno::{AxError, AxResult}; +use ax_kspin::SpinNoPreempt; use ax_lazyinit::LazyInit; use ax_memory_addr::VirtAddr; use ax_runtime::hal::{percpu::this_cpu_id, time::monotonic_time_nanos}; @@ -21,7 +23,13 @@ use crate::{ task::AsThread, }; -pub type KernelExtTracePoint = Arc>>; +// The registry entry is locked from the tracepoint fire path, which for +// `sched:sched_switch` runs inside `axtask::switch_to` (IRQ off, +// preemption disabled). A sleeping `ax_sync::Mutex` would trip the +// "sleeping in atomic context" guard there, so this lock must be a +// non-sleeping spinlock — the same kind the perf output path (`PERF_FILE`) +// uses for exactly this reason. +pub type KernelExtTracePoint = Arc>>; /// Look up a registered tracepoint by its numeric id (as found in /// `/sys/kernel/debug/tracing/events///id`). @@ -239,7 +247,7 @@ pub fn tracepoint_init() -> AxResult<()> { let ext_tps = ext_tps .into_iter() - .map(|ext_tp| (ext_tp.id(), Arc::new(Mutex::new(ext_tp)))) + .map(|ext_tp| (ext_tp.id(), Arc::new(SpinNoPreempt::new(ext_tp)))) .collect::>(); ax_println!("Initialized {} tracepoints", tp_map.len()); diff --git a/os/StarryOS/kernel/src/tracepoint/sched.rs b/os/StarryOS/kernel/src/tracepoint/sched.rs new file mode 100644 index 0000000000..25ec789a45 --- /dev/null +++ b/os/StarryOS/kernel/src/tracepoint/sched.rs @@ -0,0 +1,46 @@ +//! `sched:*` tracepoints. +//! +//! `sched_switch` is fired by `ax-task` through the cross-crate +//! [`ax_task::SchedTracepoint`] interface (gated by `tracepoint-hooks`). +//! +//! The other two `sched:*` events are defined next to their emission sites +//! rather than here: `sched_process_fork` in `crate::syscall::task::clone` +//! and `sched_process_exit` in `crate::task::ops`. Registration is by link +//! section, so their physical location does not affect discovery. + +use ax_task::SchedTracepoint; + +ktracepoint::define_event_trace!( + sched_switch, + TP_kops(crate::tracepoint::KernelTraceAux), + TP_system(sched), + TP_PROTO(prev_tid: u64, next_tid: u64, prev_state: u32), + TP_STRUCT__entry { + prev_tid: u64, + next_tid: u64, + prev_state: u32, + }, + TP_fast_assign { + prev_tid: prev_tid, + next_tid: next_tid, + prev_state: prev_state, + }, + TP_ident(__entry), + TP_printk({ + alloc::format!( + "prev_tid={} next_tid={} prev_state={}", + __entry.prev_tid, + __entry.next_tid, + __entry.prev_state, + ) + }) +); + +struct SchedTracepointImpl; + +#[ax_crate_interface::impl_interface] +impl SchedTracepoint for SchedTracepointImpl { + fn on_sched_switch(prev_tid: u64, next_tid: u64, prev_state: u32) { + trace_sched_switch(prev_tid, next_tid, prev_state); + } +} diff --git a/os/StarryOS/kernel/src/uprobe/mod.rs b/os/StarryOS/kernel/src/uprobe/mod.rs new file mode 100644 index 0000000000..6ab2903c38 --- /dev/null +++ b/os/StarryOS/kernel/src/uprobe/mod.rs @@ -0,0 +1,78 @@ +//! Per-process uprobe support. +//! +//! A uprobe plants an `int3` (or arch breakpoint) into a *user* process' text +//! and runs an eBPF program on every hit. Unlike kprobes — which share one +//! global manager because the kernel text is shared — each process owns its own +//! [`KprobeManager`](crate::kprobe::KprobeManager) / +//! [`KprobePointList`](crate::kprobe::KprobePointList) on +//! [`ProcessData`](crate::task::ProcessData), since user addresses are only +//! meaningful within one address space. +//! +//! The heavy lifting (instruction decode, out-of-line single-step) lives in the +//! `kprobe` crate; this module just routes the per-process manager and the +//! trap/debug frames into it. The user-space breakpoint insertion and the +//! out-of-line exec page come from the `KprobeAuxiliaryOps` user-mode paths in +//! [`crate::kprobe`]. + +use alloc::sync::Arc; + +use ax_task::current; +use kprobe::{ProbeBuilder, Uprobe}; + +use crate::{ + kprobe::{KernelKprobeOps, KernelRawMutex, ptregs_write_back, trapframe_to_ptregs}, + task::AsThread, +}; + +/// Concrete `kprobe::Uprobe` parameterized on the kernel's raw mutex and +/// auxiliary ops (the same `L` / `F` the kprobe types use). +pub type KernelUprobe = Uprobe; + +/// Register a uprobe into the *current* process' per-process manager. +pub fn register_uprobe(builder: ProbeBuilder) -> Arc { + let curr = current(); + let thread = curr.as_thread(); + let mut manager = thread.proc_data.uprobe_manager.lock(); + let mut point_list = thread.proc_data.uprobe_point_list.lock(); + kprobe::register_uprobe(&mut manager, &mut point_list, builder) +} + +/// Unregister a previously registered uprobe from the current process. +pub fn unregister_uprobe(uprobe: Arc) { + let curr = current(); + let thread = curr.as_thread(); + let mut manager = thread.proc_data.uprobe_manager.lock(); + let mut point_list = thread.proc_data.uprobe_point_list.lock(); + kprobe::unregister_uprobe(&mut manager, &mut point_list, uprobe); +} + +/// Dispatch a breakpoint exception to the current process' uprobe manager. +/// Returns `Some(())` if a uprobe handled it. +/// +/// Runs from exception context (IRQs disabled), so the per-process manager — a +/// sleeping mutex (see [`crate::task::ProcessData::uprobe_manager`]) — is taken +/// with `try_lock()`, which is a single CAS and safe here. At fire time the +/// manager is uncontended (arming happens in syscall context on the same task), +/// so the lock is always acquired; a contended miss just reports "unhandled". +pub fn break_uprobe_handler(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> Option<()> { + let curr = current(); + let mut manager = curr.as_thread().proc_data.uprobe_manager.try_lock()?; + let mut pt_regs = trapframe_to_ptregs(tf); + let res = kprobe::uprobe_handler_from_break(&mut manager, &mut pt_regs); + ptregs_write_back(&pt_regs, tf); + res +} + +/// Dispatch a debug (single-step) exception to the current process' uprobe +/// manager — the out-of-line step completion path on x86_64. +#[cfg(target_arch = "x86_64")] +pub fn debug_uprobe_handler(tf: &mut ax_runtime::hal::cpu::TrapFrame) -> Option<()> { + let curr = current(); + // `try_lock()` for the same reason as `break_uprobe_handler`: exception + // context, sleeping mutex. + let mut manager = curr.as_thread().proc_data.uprobe_manager.try_lock()?; + let mut pt_regs = trapframe_to_ptregs(tf); + let res = kprobe::uprobe_handler_from_debug(&mut manager, &mut pt_regs); + ptregs_write_back(&pt_regs, tf); + res +} diff --git a/os/arceos/api/axfeat/Cargo.toml b/os/arceos/api/axfeat/Cargo.toml index c3b11508c3..c696ce4c53 100644 --- a/os/arceos/api/axfeat/Cargo.toml +++ b/os/arceos/api/axfeat/Cargo.toml @@ -49,6 +49,7 @@ multitask = [ ] lockdep = ["multitask", "ax-sync/lockdep", "ax-kspin/lockdep"] task-ext = ["ax-task/task-ext"] +tracepoint-hooks = ["ax-task/tracepoint-hooks"] sched-fifo = ["ax-task/sched-fifo"] sched-rr = ["ax-task/sched-rr", "irq"] sched-cfs = ["ax-task/sched-cfs", "irq"] diff --git a/os/arceos/modules/axtask/Cargo.toml b/os/arceos/modules/axtask/Cargo.toml index 29cbe82f61..5a743d5804 100644 --- a/os/arceos/modules/axtask/Cargo.toml +++ b/os/arceos/modules/axtask/Cargo.toml @@ -39,6 +39,12 @@ lockdep = [ ] task-ext = ["dep:extern-trait"] tls = ["ax-hal/tls"] +# Cross-crate tracepoint hook for the scheduler context switch. +# Enabling this exposes a `SchedTracepoint` interface (via `ax-crate-interface`) +# that another crate (e.g. starry-kernel) must implement; otherwise the link +# step will fail with an unresolved symbol. Pure ArceOS unikernels leave this +# off and pay no cost. +tracepoint-hooks = ["multitask"] uspace = ["ax-hal/uspace"] sched-cfs = ["multitask", "preempt"] diff --git a/os/arceos/modules/axtask/src/lib.rs b/os/arceos/modules/axtask/src/lib.rs index d6f8257677..3b37d91dc1 100644 --- a/os/arceos/modules/axtask/src/lib.rs +++ b/os/arceos/modules/axtask/src/lib.rs @@ -68,6 +68,8 @@ cfg_if::cfg_if! { mod api; #[cfg(feature = "lockdep")] mod lockdep; + #[cfg(feature = "tracepoint-hooks")] + mod sched_tracepoint; mod wait_queue; #[cfg(feature = "irq")] @@ -79,6 +81,8 @@ cfg_if::cfg_if! { #[cfg_attr(doc, doc(cfg(feature = "multitask")))] pub use self::api::*; pub use self::api::{sleep, sleep_until, yield_now}; + #[cfg(feature = "tracepoint-hooks")] + pub use self::sched_tracepoint::SchedTracepoint; } else { mod api_s; pub use self::api_s::{sleep, sleep_until, yield_now}; diff --git a/os/arceos/modules/axtask/src/run_queue.rs b/os/arceos/modules/axtask/src/run_queue.rs index 36dc37456c..0d31ff3af4 100644 --- a/os/arceos/modules/axtask/src/run_queue.rs +++ b/os/arceos/modules/axtask/src/run_queue.rs @@ -704,6 +704,18 @@ impl AxRunQueue { } } + // `prev_task.state()` must be sampled before the architectural switch: + // callers like `exit_current` already set it to `Exited`/`Blocked`, + // and that pre-switch state is what `sched:sched_switch` reports. + #[cfg(feature = "tracepoint-hooks")] + ax_crate_interface::call_interface!( + crate::sched_tracepoint::SchedTracepoint::on_sched_switch( + prev_task.id().as_u64(), + next_task.id().as_u64(), + prev_task.state() as u32, + ) + ); + unsafe { let prev_ctx_ptr = prev_task.ctx_mut_ptr(); let next_ctx_ptr = next_task.ctx_mut_ptr(); diff --git a/os/arceos/modules/axtask/src/sched_tracepoint.rs b/os/arceos/modules/axtask/src/sched_tracepoint.rs new file mode 100644 index 0000000000..18be462aa3 --- /dev/null +++ b/os/arceos/modules/axtask/src/sched_tracepoint.rs @@ -0,0 +1,13 @@ +//! Cross-crate scheduler tracepoint hook. +//! +//! Builds that enable `tracepoint-hooks` without a matching +//! `#[ax_crate_interface::impl_interface]` implementor will fail at link. + +/// `prev_state` is the [`crate::task::TaskState`] discriminant; implementors +/// decode without depending on the enum. +#[ax_crate_interface::def_interface] +pub trait SchedTracepoint { + /// Fired from `switch_to` after the `prev == next` short-circuit and + /// before the architectural context switch. IRQs are disabled. + fn on_sched_switch(prev_tid: u64, next_tid: u64, prev_state: u32); +}