diff --git a/os/StarryOS/kernel/src/ebpf/mod.rs b/os/StarryOS/kernel/src/ebpf/mod.rs index e861046945..a334144310 100644 --- a/os/StarryOS/kernel/src/ebpf/mod.rs +++ b/os/StarryOS/kernel/src/ebpf/mod.rs @@ -51,6 +51,12 @@ use crate::{ /// without re-running the kbpf-basic init on every call. pub static BPF_HELPER_FUN_SET: LazyInit> = LazyInit::new(); +/// BPF helper ID for `bpf_probe_read` (legacy, address-space-agnostic). +const BPF_FUNC_PROBE_READ: u32 = 4; +/// BPF helper ID for `bpf_probe_read_kernel`. kbpf-basic only registers +/// `bpf_probe_read` so we alias 113 onto the same raw helper. +const BPF_FUNC_PROBE_READ_KERNEL: u32 = 113; + /// Initialize the BPF subsystem: build the helper-function table from /// `kbpf-basic`. Must be called before `sys_bpf(BPF_PROG_LOAD)` so loaded /// programs can resolve the helper ids referenced in their instructions. @@ -61,8 +67,8 @@ pub fn init_ebpf() { // tracepoint program uses to pull fields out of its sample buffer. // kbpf-basic only registers the legacy `bpf_probe_read` (id 4), whose raw // reader is address-space-agnostic in this VM, so alias 113 onto it. - if let Some(&probe_read) = set.get(&4) { - set.entry(113).or_insert(probe_read); + if let Some(&probe_read) = set.get(&BPF_FUNC_PROBE_READ) { + set.entry(BPF_FUNC_PROBE_READ_KERNEL).or_insert(probe_read); } BPF_HELPER_FUN_SET.init_once(set); } diff --git a/os/StarryOS/kernel/src/perf/bpf.rs b/os/StarryOS/kernel/src/perf/bpf.rs index 0898cc0111..4e32bac946 100644 --- a/os/StarryOS/kernel/src/perf/bpf.rs +++ b/os/StarryOS/kernel/src/perf/bpf.rs @@ -23,7 +23,7 @@ use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr}; use ax_task::IrqNotify; use axpoll::{IoEvents, PollSet, Pollable}; use kbpf_basic::{ - linux_bpf::perf_event_sample_format, + linux_bpf::{perf_event_header, perf_event_mmap_page, perf_event_sample_format}, perf::{PerfProbeArgs, bpf::BpfPerfEvent}, }; use kprobe::PtRegs; @@ -37,6 +37,11 @@ use crate::{ file::FileLike, }; +/// Number of 4K pages reserved for x86_64 BPF JIT executable memory. +/// Each JIT-compiled eBPF program fits within this space; the allocation +/// is sized generously (16 KiB) since programs are typically < 1 page. +const BPF_JIT_MEM_PAGES: usize = 4; + /// Wraps `kbpf_basic::perf::bpf::BpfPerfEvent` with kernel state: a poll /// set so readers can wait for new records, and a weak handle to the /// backing pages produced by `device_mmap` (Some after the first @@ -72,6 +77,9 @@ pub struct BpfPerfEventWrapper { /// mapping still exists. See the type-level docs for the ownership /// rationale. pages: Option>, + /// Kernel virtual address of the ringbuf start, saved during + /// `device_mmap` for `read(2)` access. + mmap_kvirt: Option, } impl BpfPerfEventWrapper { @@ -87,6 +95,7 @@ impl BpfPerfEventWrapper { poll_notify, poll_alive, pages: None, + mmap_kvirt: None, } } @@ -113,6 +122,77 @@ impl BpfPerfEventWrapper { } Ok(()) } + + /// Read one ringbuf record into `dst`, returning the number of bytes + /// copied (including the `perf_event_header`). Returns `Ok(0)` when no + /// data is available. Safety: the ringbuf pages are valid as long as + /// the kernel virtual address was saved during `device_mmap` and the + /// strong page ref still exists (checked via `is_mapped`). + pub(crate) fn try_read_record(&self, dst: &mut [u8]) -> AxResult { + let kvirt = self.mmap_kvirt.ok_or(AxError::NotConnected)?; + if !self.is_mapped() { + return Ok(0); + } + + // SAFETY: the ringbuf pages are pinned by the VMA and zero- + // initialised; the `perf_event_mmap_page` at offset 0 was + // initialised by `BpfPerfEvent::do_mmap` during `device_mmap`. + let mmap = unsafe { &*(kvirt as *const perf_event_mmap_page) }; + if mmap.data_tail == mmap.data_head { + return Ok(0); + } + + let data_region_size = mmap.data_size as usize; + let tail = (mmap.data_tail as usize) % data_region_size; + + // SAFETY: the data region starts at PAGE_SIZE offset into the + // ringbuf allocation; `data_size` was set by kbpf-basic during + // `do_mmap` and bounds the accessible region. + let data_region = unsafe { + core::slice::from_raw_parts((kvirt + PAGE_SIZE_4K) as *const u8, data_region_size) + }; + + let hdr_size = core::mem::size_of::(); + if hdr_size > dst.len() { + return Err(AxError::InvalidInput); + } + + // Read the perf_event_header (handling ring-wrap). + let hdr_end = wrapping_add(tail, hdr_size, data_region_size); + copy_ring(data_region, tail, hdr_end, &mut dst[..hdr_size]); + + // SAFETY: the header bytes are a valid `perf_event_header` written + // by kbpf-basic's `RingPage::write_sample`. + let header = unsafe { &*(dst.as_ptr() as *const perf_event_header) }; + let record_size = header.size as usize; + + if record_size < hdr_size || record_size > dst.len() { + warn!( + "eBPF perf: bad record size {record_size} (hdr_size={hdr_size}, buf_len={})", + dst.len() + ); + return Err(AxError::InvalidInput); + } + + // Copy the full record (header already copied at offset 0). + if record_size > hdr_size { + let payload_start = wrapping_add(tail, hdr_size, data_region_size); + let payload_end = wrapping_add(tail, record_size, data_region_size); + copy_ring( + data_region, + payload_start, + payload_end, + &mut dst[hdr_size..record_size], + ); + } + + // Advance data_tail past this record. + // SAFETY: the mmap page is writable by the kernel. + let mmap_mut = unsafe { &mut *(kvirt as *mut perf_event_mmap_page) }; + mmap_mut.data_tail += record_size as u64; + + Ok(record_size) + } } impl Drop for BpfPerfEventWrapper { @@ -146,6 +226,35 @@ impl Debug for BpfPerfEventWrapper { } } +/// Compute `(base + offset) % modulus`. The modulus is always the ring +/// data-region size so the result fits in a single wrapping-modulo round. +#[inline] +fn wrapping_add(base: usize, offset: usize, modulus: usize) -> usize { + debug_assert!(modulus > 0); + let sum = base + offset; + if sum < modulus { sum } else { sum % modulus } +} + +/// Copy bytes from a ring buffer (`src`) into a linear buffer (`dst`). +/// The ring extent is `[from, to)` with `from` and `to` modulo-wrapped into +/// the ring size. Handles the zero or one wrap case. +fn copy_ring(src: &[u8], from: usize, to: usize, dst: &mut [u8]) { + let ring_size = src.len(); + let len = to.wrapping_sub(from) % ring_size; + if len == 0 { + return; + } + debug_assert!(len <= dst.len(), "copy_ring: dst too small"); + let end = from + len; + if end <= ring_size { + dst[..len].copy_from_slice(&src[from..end]); + } else { + let first = ring_size - from; + dst[..first].copy_from_slice(&src[from..]); + dst[first..len].copy_from_slice(&src[..(end % ring_size)]); + } +} + impl PerfEventOps for BpfPerfEventWrapper { fn enable(&mut self) -> AxResult<()> { self.inner.enable().into_ax_result()?; @@ -196,6 +305,7 @@ impl PerfEventOps for BpfPerfEventWrapper { // (see the type-level docs). Without the anchor the pages would free // under a live mapping. self.pages = Some(Arc::downgrade(&pages)); + self.mmap_kvirt = Some(kvirt.as_usize()); let anchor: Arc = pages; Ok((paddr, anchor)) } @@ -273,17 +383,23 @@ impl OwnedEbpfVm { let _ = vm.register_helper(*key, *value); } } - // TODO: not all of the address space is accessible to a BPF program; - // allowing the full `0..u64::MAX` range disables rbpf's bounds check - // and lets a buggy/hostile program read arbitrary kernel memory via - // direct loads. Narrow this to the legitimately-reachable context / - // map / stack ranges once kbpf-basic exposes the per-program bounds. - vm.register_allowed_memory(0..u64::MAX); + // bpftrace / libbpf / aya observability programs access kernel + // memory exclusively through registered helper functions (e.g. + // bpf_probe_read, bpf_map_lookup_elem), never via direct LD_ABS + // / LD_IND / LD_DW loads. We intentionally do NOT call + // `register_allowed_memory` so rbpf's bounds check rejects any + // direct memory access, preventing a buggy or hostile program + // from reading arbitrary kernel memory. + // + // If a future program legitimately needs direct loads (e.g. a + // raw socket filter that touches packet data), register its + // specific buffer range here rather than re-opening the full + // address space. #[cfg(target_arch = "x86_64")] { // TODO: calculate a more precise size. - let mut jit_exec_memory = BPFJitMemory::new(4)?; + let mut jit_exec_memory = BPFJitMemory::new(BPF_JIT_MEM_PAGES)?; // SAFETY: `jit_exec_memory` is moved into the returned // `OwnedEbpfVm` after `vm`; field drop order guarantees `vm` is // destroyed before the executable mapping is unmapped. diff --git a/os/StarryOS/kernel/src/perf/kprobe.rs b/os/StarryOS/kernel/src/perf/kprobe.rs index c66d3cad18..d29a3e1047 100644 --- a/os/StarryOS/kernel/src/perf/kprobe.rs +++ b/os/StarryOS/kernel/src/perf/kprobe.rs @@ -16,6 +16,14 @@ use axpoll::Pollable; use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig}; use kprobe::{CallBackFunc, KretprobeBuilder, ProbeBuilder, PtRegs}; +/// Config value for entry probes (kprobe/uprobe), per Linux PERF_TYPE_PROBE ABI. +pub const PROBE_CONFIG_ENTRY: u64 = 0; +/// Config value for return probes (kretprobe/uretprobe), per Linux PERF_TYPE_PROBE ABI. +pub const PROBE_CONFIG_RETURN: u64 = 1; +/// Maximum number of concurrently active kretprobe instances, matching +/// Linux's `max(10, 2*NR_CPUS)` default for single-CPU configurations. +const KRETPROBE_MAX_ACTIVE: u32 = 10; + use crate::{ file::FileLike, kprobe::{ @@ -182,25 +190,24 @@ fn perf_probe_arg_to_kretprobe_builder( ) -> AxResult> { let symbol = &args.name; let addr = lookup_symbol_addr(symbol)?; - Ok(KretprobeBuilder::::new(10) - .with_symbol(symbol.clone()) - .with_symbol_addr(addr)) + Ok( + KretprobeBuilder::::new(KRETPROBE_MAX_ACTIVE) + .with_symbol(symbol.clone()) + .with_symbol_addr(addr), + ) } /// Build a `ProbePerfEvent` for a `PERF_TYPE_KPROBE` perf_event_open call. -/// Config 0 = kprobe; config 1 = kretprobe (per kbpf-basic convention). +/// Config `PROBE_CONFIG_ENTRY` (0) = kprobe; `PROBE_CONFIG_RETURN` (1) = kretprobe. pub fn perf_event_open_kprobe(args: PerfProbeArgs) -> AxResult { let probe = match args.config { - PerfProbeConfig::Raw(val) => { - if val == 0 { - let builder = perf_probe_arg_to_kprobe_builder(&args)?; - ProbeTy::Kprobe(register_kprobe(builder)) - } else if val == 1 { - let builder = perf_probe_arg_to_kretprobe_builder(&args)?; - ProbeTy::Kretprobe(register_kretprobe(builder)) - } else { - return Err(AxError::InvalidInput); - } + PerfProbeConfig::Raw(PROBE_CONFIG_ENTRY) => { + let builder = perf_probe_arg_to_kprobe_builder(&args)?; + ProbeTy::Kprobe(register_kprobe(builder)) + } + PerfProbeConfig::Raw(PROBE_CONFIG_RETURN) => { + let builder = perf_probe_arg_to_kretprobe_builder(&args)?; + ProbeTy::Kretprobe(register_kretprobe(builder)) } _ => return Err(AxError::InvalidInput), }; diff --git a/os/StarryOS/kernel/src/perf/mod.rs b/os/StarryOS/kernel/src/perf/mod.rs index fb41bf449b..1f2734c314 100644 --- a/os/StarryOS/kernel/src/perf/mod.rs +++ b/os/StarryOS/kernel/src/perf/mod.rs @@ -13,7 +13,12 @@ pub mod tracepoint; pub mod uprobe; use alloc::{borrow::Cow, boxed::Box, sync::Arc, vec}; -use core::{any::Any, ffi::c_void, fmt::Debug}; +use core::{ + any::Any, + ffi::c_void, + fmt::Debug, + sync::atomic::{AtomicBool, Ordering}, +}; use ax_errno::{AxError, AxResult}; use ax_io::Read; @@ -21,7 +26,8 @@ use ax_kspin::{SpinNoPreempt, SpinNoPreemptGuard}; use ax_lazyinit::LazyInit; use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr, PhysAddrRange, VirtAddr, VirtAddrRange}; use ax_runtime::hal::paging::MappingFlags; -use axpoll::Pollable; +use ax_task::future::{block_on, poll_io}; +use axpoll::{IoEvents, Pollable}; pub use bpf::BpfPerfEventWrapper; use hashbrown::HashMap; use kbpf_basic::{ @@ -73,6 +79,9 @@ pub trait PerfEventOps: Pollable + Send + Sync + Debug { /// `Box` so the inner implementation can stay generic. pub struct PerfEvent { event: SpinNoPreempt>, + /// O_NONBLOCK flag. When true, `read(2)` returns EAGAIN instead of + /// blocking on an empty ringbuf. + nonblocking: AtomicBool, } impl Debug for PerfEvent { @@ -86,6 +95,7 @@ impl PerfEvent { pub fn new(event: Box) -> Self { PerfEvent { event: SpinNoPreempt::new(event), + nonblocking: AtomicBool::new(false), } } @@ -105,9 +115,30 @@ impl Pollable for PerfEvent { } } +/// Stack buffer size for reading a single perf ringbuf record. The +/// ringbuf data region minimum is one page, and a single perf record +/// always fits within one data-region page. +const PERF_READ_BUF_SIZE: usize = PAGE_SIZE_4K; + impl FileLike for PerfEvent { - fn read(&self, _dst: &mut crate::file::IoDst) -> AxResult { - Err(AxError::Unsupported) + fn read(&self, dst: &mut crate::file::IoDst) -> AxResult { + block_on(poll_io(self, IoEvents::IN, self.nonblocking(), || { + let mut event = self.event.lock(); + let bpf_event = event + .as_any_mut() + .downcast_mut::() + .ok_or(AxError::Unsupported)?; + let mut buf = [0u8; PERF_READ_BUF_SIZE]; + let n = bpf_event.try_read_record(&mut buf)?; + // Release the lock before writing to dst (I/O outside the spin + // lock) and before returning WouldBlock (the lock must not be + // held across the poll_io reschedule). + drop(event); + if n == 0 { + return Err(AxError::WouldBlock); + } + dst.write(&buf[..n]).map_err(|_| AxError::Io) + })) } fn write(&self, _src: &mut crate::file::IoSrc) -> AxResult { @@ -157,6 +188,15 @@ impl FileLike for PerfEvent { Some(anchor), )) } + + fn nonblocking(&self) -> bool { + self.nonblocking.load(Ordering::Acquire) + } + + fn set_nonblocking(&self, on: bool) -> AxResult { + self.nonblocking.store(on, Ordering::Release); + Ok(()) + } } /// `perf_event_open(2)` syscall entry. Copies the user `perf_event_attr` in diff --git a/os/StarryOS/kernel/src/perf/uprobe.rs b/os/StarryOS/kernel/src/perf/uprobe.rs index 84374b2775..428ce33ae0 100644 --- a/os/StarryOS/kernel/src/perf/uprobe.rs +++ b/os/StarryOS/kernel/src/perf/uprobe.rs @@ -15,7 +15,7 @@ use ax_errno::{AxError, AxResult}; use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig}; use kprobe::ProbeBuilder; -use super::kprobe::{ProbePerfEvent, ProbeTy}; +use super::kprobe::{PROBE_CONFIG_ENTRY, PROBE_CONFIG_RETURN, ProbePerfEvent, ProbeTy}; use crate::{ kprobe::KprobeAuxiliary, task::{AsThread, get_task}, @@ -74,13 +74,13 @@ fn perf_probe_arg_to_uprobe_builder( /// Build a uprobe perf event from `perf_event_open` args. pub fn perf_event_open_uprobe(args: PerfProbeArgs) -> AxResult { let probe = match args.config { - PerfProbeConfig::Raw(0) => { + PerfProbeConfig::Raw(PROBE_CONFIG_ENTRY) => { let builder = perf_probe_arg_to_uprobe_builder(&args)?; ProbeTy::Uprobe(crate::uprobe::register_uprobe(builder)) } - PerfProbeConfig::Raw(1) => { + PerfProbeConfig::Raw(PROBE_CONFIG_RETURN) => { // uretprobe — not implemented for user space yet. - warn!("uprobe: uretprobe (config=1) is not yet supported"); + warn!("uprobe: uretprobe is not yet supported"); return Err(AxError::Unsupported); } other => { diff --git a/os/StarryOS/kernel/src/pseudofs/dev/card1.rs b/os/StarryOS/kernel/src/pseudofs/dev/card1.rs index d80ef1b702..f06158525b 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/card1.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/card1.rs @@ -194,7 +194,8 @@ impl DeviceOps for Card1 { } _ => { - panic!("card1: unsupported ioctl nr {nr:#x}"); + warn!("card1: unsupported ioctl nr {nr:#x}"); + return Err(VfsError::OperationNotSupported); } } copy_to_user(arg as _, stack_data.as_mut_ptr(), out_size)?; diff --git a/os/StarryOS/kernel/src/tracepoint/mod.rs b/os/StarryOS/kernel/src/tracepoint/mod.rs index 488a420bf5..8ede91f25e 100644 --- a/os/StarryOS/kernel/src/tracepoint/mod.rs +++ b/os/StarryOS/kernel/src/tracepoint/mod.rs @@ -27,6 +27,11 @@ use crate::{ task::AsThread, }; +/// Maximum number of trace records kept in the raw trace pipe ring buffer. +const TRACE_RAW_PIPE_CAPACITY: usize = 4096; +/// Maximum number of PID→cmdline entries in the command-line cache. +const TRACE_CMDLINE_CACHE_SIZE: usize = 4096; + // The registry entry is locked from the tracepoint fire path, which for // `sched:sched_switch` runs inside `axtask::switch_to` (IRQ off, // preemption disabled). A sleeping `ax_sync::Mutex` would trip the @@ -71,7 +76,7 @@ impl TraceState { const fn new() -> Self { Self { point_map: LazyInit::new(), - raw_pipe: Mutex::new(TracePipeRaw::new(4096)), + raw_pipe: Mutex::new(TracePipeRaw::new(TRACE_RAW_PIPE_CAPACITY)), pipe_event: PollSet::new(), pipe_notify: IrqNotify::new(), cmdline_cache: LazyInit::new(), @@ -277,7 +282,7 @@ pub fn tracepoint_init() -> AxResult<()> { TRACE_STATE .cmdline_cache .init_once(Mutex::new(TraceCmdLineCache::new( - NonZero::new(4096).unwrap(), + NonZero::new(TRACE_CMDLINE_CACHE_SIZE).unwrap(), ))); start_trace_pipe_notify_worker(); Ok(())