Skip to content
Closed
10 changes: 8 additions & 2 deletions os/StarryOS/kernel/src/ebpf/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ use crate::{
/// without re-running the kbpf-basic init on every call.
pub static BPF_HELPER_FUN_SET: LazyInit<BTreeMap<u32, RawBPFHelperFn>> = LazyInit::new();

/// BPF helper ID for `bpf_probe_read` (legacy, address-space-agnostic).
const BPF_FUNC_PROBE_READ: u32 = 4;
/// BPF helper ID for `bpf_probe_read_kernel`. kbpf-basic only registers
/// `bpf_probe_read` so we alias 113 onto the same raw helper.
const BPF_FUNC_PROBE_READ_KERNEL: u32 = 113;

/// Initialize the BPF subsystem: build the helper-function table from
/// `kbpf-basic`. Must be called before `sys_bpf(BPF_PROG_LOAD)` so loaded
/// programs can resolve the helper ids referenced in their instructions.
Expand All @@ -61,8 +67,8 @@ pub fn init_ebpf() {
// tracepoint program uses to pull fields out of its sample buffer.
// kbpf-basic only registers the legacy `bpf_probe_read` (id 4), whose raw
// reader is address-space-agnostic in this VM, so alias 113 onto it.
if let Some(&probe_read) = set.get(&4) {
set.entry(113).or_insert(probe_read);
if let Some(&probe_read) = set.get(&BPF_FUNC_PROBE_READ) {
set.entry(BPF_FUNC_PROBE_READ_KERNEL).or_insert(probe_read);
}
BPF_HELPER_FUN_SET.init_once(set);
}
Expand Down
132 changes: 124 additions & 8 deletions os/StarryOS/kernel/src/perf/bpf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr};
use ax_task::IrqNotify;
use axpoll::{IoEvents, PollSet, Pollable};
use kbpf_basic::{
linux_bpf::perf_event_sample_format,
linux_bpf::{perf_event_header, perf_event_mmap_page, perf_event_sample_format},
perf::{PerfProbeArgs, bpf::BpfPerfEvent},
};
use kprobe::PtRegs;
Expand All @@ -37,6 +37,11 @@ use crate::{
file::FileLike,
};

/// Number of 4K pages reserved for x86_64 BPF JIT executable memory.
/// Each JIT-compiled eBPF program fits within this space; the allocation
/// is sized generously (16 KiB) since programs are typically < 1 page.
const BPF_JIT_MEM_PAGES: usize = 4;

/// Wraps `kbpf_basic::perf::bpf::BpfPerfEvent` with kernel state: a poll
/// set so readers can wait for new records, and a weak handle to the
/// backing pages produced by `device_mmap` (Some after the first
Expand Down Expand Up @@ -72,6 +77,9 @@ pub struct BpfPerfEventWrapper {
/// mapping still exists. See the type-level docs for the ownership
/// rationale.
pages: Option<Weak<GlobalPage>>,
/// Kernel virtual address of the ringbuf start, saved during
/// `device_mmap` for `read(2)` access.
mmap_kvirt: Option<usize>,
}

impl BpfPerfEventWrapper {
Expand All @@ -87,6 +95,7 @@ impl BpfPerfEventWrapper {
poll_notify,
poll_alive,
pages: None,
mmap_kvirt: None,
}
}

Expand All @@ -113,6 +122,77 @@ impl BpfPerfEventWrapper {
}
Ok(())
}

/// Read one ringbuf record into `dst`, returning the number of bytes
/// copied (including the `perf_event_header`). Returns `Ok(0)` when no
/// data is available. Safety: the ringbuf pages are valid as long as
/// the kernel virtual address was saved during `device_mmap` and the
/// strong page ref still exists (checked via `is_mapped`).
pub(crate) fn try_read_record(&self, dst: &mut [u8]) -> AxResult<usize> {
let kvirt = self.mmap_kvirt.ok_or(AxError::NotConnected)?;
if !self.is_mapped() {
return Ok(0);
}

// SAFETY: the ringbuf pages are pinned by the VMA and zero-
// initialised; the `perf_event_mmap_page` at offset 0 was
// initialised by `BpfPerfEvent::do_mmap` during `device_mmap`.
let mmap = unsafe { &*(kvirt as *const perf_event_mmap_page) };
if mmap.data_tail == mmap.data_head {
return Ok(0);
}

let data_region_size = mmap.data_size as usize;
let tail = (mmap.data_tail as usize) % data_region_size;

// SAFETY: the data region starts at PAGE_SIZE offset into the
// ringbuf allocation; `data_size` was set by kbpf-basic during
// `do_mmap` and bounds the accessible region.
let data_region = unsafe {
core::slice::from_raw_parts((kvirt + PAGE_SIZE_4K) as *const u8, data_region_size)
};

let hdr_size = core::mem::size_of::<perf_event_header>();
if hdr_size > dst.len() {
return Err(AxError::InvalidInput);
}

// Read the perf_event_header (handling ring-wrap).
let hdr_end = wrapping_add(tail, hdr_size, data_region_size);
copy_ring(data_region, tail, hdr_end, &mut dst[..hdr_size]);

// SAFETY: the header bytes are a valid `perf_event_header` written
// by kbpf-basic's `RingPage::write_sample`.
let header = unsafe { &*(dst.as_ptr() as *const perf_event_header) };
let record_size = header.size as usize;

if record_size < hdr_size || record_size > dst.len() {
warn!(
"eBPF perf: bad record size {record_size} (hdr_size={hdr_size}, buf_len={})",
dst.len()
);
return Err(AxError::InvalidInput);
}

// Copy the full record (header already copied at offset 0).
if record_size > hdr_size {
let payload_start = wrapping_add(tail, hdr_size, data_region_size);
let payload_end = wrapping_add(tail, record_size, data_region_size);
copy_ring(
data_region,
payload_start,
payload_end,
&mut dst[hdr_size..record_size],
);
}

// Advance data_tail past this record.
// SAFETY: the mmap page is writable by the kernel.
let mmap_mut = unsafe { &mut *(kvirt as *mut perf_event_mmap_page) };
mmap_mut.data_tail += record_size as u64;

Ok(record_size)
}
}

impl Drop for BpfPerfEventWrapper {
Expand Down Expand Up @@ -146,6 +226,35 @@ impl Debug for BpfPerfEventWrapper {
}
}

/// Compute `(base + offset) % modulus`. The modulus is always the ring
/// data-region size so the result fits in a single wrapping-modulo round.
#[inline]
fn wrapping_add(base: usize, offset: usize, modulus: usize) -> usize {
debug_assert!(modulus > 0);
let sum = base + offset;
if sum < modulus { sum } else { sum % modulus }
}

/// Copy bytes from a ring buffer (`src`) into a linear buffer (`dst`).
/// The ring extent is `[from, to)` with `from` and `to` modulo-wrapped into
/// the ring size. Handles the zero or one wrap case.
fn copy_ring(src: &[u8], from: usize, to: usize, dst: &mut [u8]) {
let ring_size = src.len();
let len = to.wrapping_sub(from) % ring_size;
if len == 0 {
return;
}
debug_assert!(len <= dst.len(), "copy_ring: dst too small");
let end = from + len;
if end <= ring_size {
dst[..len].copy_from_slice(&src[from..end]);
} else {
let first = ring_size - from;
dst[..first].copy_from_slice(&src[from..]);
dst[first..len].copy_from_slice(&src[..(end % ring_size)]);
}
}

impl PerfEventOps for BpfPerfEventWrapper {
fn enable(&mut self) -> AxResult<()> {
self.inner.enable().into_ax_result()?;
Expand Down Expand Up @@ -196,6 +305,7 @@ impl PerfEventOps for BpfPerfEventWrapper {
// (see the type-level docs). Without the anchor the pages would free
// under a live mapping.
self.pages = Some(Arc::downgrade(&pages));
self.mmap_kvirt = Some(kvirt.as_usize());
let anchor: Arc<dyn Any + Send + Sync> = pages;
Ok((paddr, anchor))
}
Expand Down Expand Up @@ -273,17 +383,23 @@ impl OwnedEbpfVm {
let _ = vm.register_helper(*key, *value);
}
}
// TODO: not all of the address space is accessible to a BPF program;
// allowing the full `0..u64::MAX` range disables rbpf's bounds check
// and lets a buggy/hostile program read arbitrary kernel memory via
// direct loads. Narrow this to the legitimately-reachable context /
// map / stack ranges once kbpf-basic exposes the per-program bounds.
vm.register_allowed_memory(0..u64::MAX);
// bpftrace / libbpf / aya observability programs access kernel
// memory exclusively through registered helper functions (e.g.
// bpf_probe_read, bpf_map_lookup_elem), never via direct LD_ABS
// / LD_IND / LD_DW loads. We intentionally do NOT call
// `register_allowed_memory` so rbpf's bounds check rejects any
// direct memory access, preventing a buggy or hostile program
// from reading arbitrary kernel memory.
//
// If a future program legitimately needs direct loads (e.g. a
// raw socket filter that touches packet data), register its
// specific buffer range here rather than re-opening the full
// address space.

#[cfg(target_arch = "x86_64")]
{
// TODO: calculate a more precise size.
let mut jit_exec_memory = BPFJitMemory::new(4)?;
let mut jit_exec_memory = BPFJitMemory::new(BPF_JIT_MEM_PAGES)?;
// SAFETY: `jit_exec_memory` is moved into the returned
// `OwnedEbpfVm` after `vm`; field drop order guarantees `vm` is
// destroyed before the executable mapping is unmapped.
Expand Down
35 changes: 21 additions & 14 deletions os/StarryOS/kernel/src/perf/kprobe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ use axpoll::Pollable;
use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig};
use kprobe::{CallBackFunc, KretprobeBuilder, ProbeBuilder, PtRegs};

/// Config value for entry probes (kprobe/uprobe), per Linux PERF_TYPE_PROBE ABI.
pub const PROBE_CONFIG_ENTRY: u64 = 0;
/// Config value for return probes (kretprobe/uretprobe), per Linux PERF_TYPE_PROBE ABI.
pub const PROBE_CONFIG_RETURN: u64 = 1;
/// Maximum number of concurrently active kretprobe instances, matching
/// Linux's `max(10, 2*NR_CPUS)` default for single-CPU configurations.
const KRETPROBE_MAX_ACTIVE: u32 = 10;

use crate::{
file::FileLike,
kprobe::{
Expand Down Expand Up @@ -182,25 +190,24 @@ fn perf_probe_arg_to_kretprobe_builder(
) -> AxResult<KretprobeBuilder<KernelRawMutex>> {
let symbol = &args.name;
let addr = lookup_symbol_addr(symbol)?;
Ok(KretprobeBuilder::<KernelRawMutex>::new(10)
.with_symbol(symbol.clone())
.with_symbol_addr(addr))
Ok(
KretprobeBuilder::<KernelRawMutex>::new(KRETPROBE_MAX_ACTIVE)
.with_symbol(symbol.clone())
.with_symbol_addr(addr),
)
}

/// Build a `ProbePerfEvent` for a `PERF_TYPE_KPROBE` perf_event_open call.
/// Config 0 = kprobe; config 1 = kretprobe (per kbpf-basic convention).
/// Config `PROBE_CONFIG_ENTRY` (0) = kprobe; `PROBE_CONFIG_RETURN` (1) = kretprobe.
pub fn perf_event_open_kprobe(args: PerfProbeArgs) -> AxResult<ProbePerfEvent> {
let probe = match args.config {
PerfProbeConfig::Raw(val) => {
if val == 0 {
let builder = perf_probe_arg_to_kprobe_builder(&args)?;
ProbeTy::Kprobe(register_kprobe(builder))
} else if val == 1 {
let builder = perf_probe_arg_to_kretprobe_builder(&args)?;
ProbeTy::Kretprobe(register_kretprobe(builder))
} else {
return Err(AxError::InvalidInput);
}
PerfProbeConfig::Raw(PROBE_CONFIG_ENTRY) => {
let builder = perf_probe_arg_to_kprobe_builder(&args)?;
ProbeTy::Kprobe(register_kprobe(builder))
}
PerfProbeConfig::Raw(PROBE_CONFIG_RETURN) => {
let builder = perf_probe_arg_to_kretprobe_builder(&args)?;
ProbeTy::Kretprobe(register_kretprobe(builder))
}
_ => return Err(AxError::InvalidInput),
};
Expand Down
48 changes: 44 additions & 4 deletions os/StarryOS/kernel/src/perf/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,21 @@ pub mod tracepoint;
pub mod uprobe;

use alloc::{borrow::Cow, boxed::Box, sync::Arc, vec};
use core::{any::Any, ffi::c_void, fmt::Debug};
use core::{
any::Any,
ffi::c_void,
fmt::Debug,
sync::atomic::{AtomicBool, Ordering},
};

use ax_errno::{AxError, AxResult};
use ax_io::Read;
use ax_kspin::{SpinNoPreempt, SpinNoPreemptGuard};
use ax_lazyinit::LazyInit;
use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr, PhysAddrRange, VirtAddr, VirtAddrRange};
use ax_runtime::hal::paging::MappingFlags;
use axpoll::Pollable;
use ax_task::future::{block_on, poll_io};
use axpoll::{IoEvents, Pollable};
pub use bpf::BpfPerfEventWrapper;
use hashbrown::HashMap;
use kbpf_basic::{
Expand Down Expand Up @@ -73,6 +79,9 @@ pub trait PerfEventOps: Pollable + Send + Sync + Debug {
/// `Box<dyn PerfEventOps>` so the inner implementation can stay generic.
pub struct PerfEvent {
event: SpinNoPreempt<Box<dyn PerfEventOps>>,
/// O_NONBLOCK flag. When true, `read(2)` returns EAGAIN instead of
/// blocking on an empty ringbuf.
nonblocking: AtomicBool,
}

impl Debug for PerfEvent {
Expand All @@ -86,6 +95,7 @@ impl PerfEvent {
pub fn new(event: Box<dyn PerfEventOps>) -> Self {
PerfEvent {
event: SpinNoPreempt::new(event),
nonblocking: AtomicBool::new(false),
}
}

Expand All @@ -105,9 +115,30 @@ impl Pollable for PerfEvent {
}
}

/// Stack buffer size for reading a single perf ringbuf record. The
/// ringbuf data region minimum is one page, and a single perf record
/// always fits within one data-region page.
const PERF_READ_BUF_SIZE: usize = PAGE_SIZE_4K;

impl FileLike for PerfEvent {
fn read(&self, _dst: &mut crate::file::IoDst) -> AxResult<usize> {
Err(AxError::Unsupported)
fn read(&self, dst: &mut crate::file::IoDst) -> AxResult<usize> {
block_on(poll_io(self, IoEvents::IN, self.nonblocking(), || {
let mut event = self.event.lock();
let bpf_event = event
.as_any_mut()
.downcast_mut::<BpfPerfEventWrapper>()
.ok_or(AxError::Unsupported)?;
let mut buf = [0u8; PERF_READ_BUF_SIZE];
let n = bpf_event.try_read_record(&mut buf)?;
// Release the lock before writing to dst (I/O outside the spin
// lock) and before returning WouldBlock (the lock must not be
// held across the poll_io reschedule).
drop(event);
if n == 0 {
return Err(AxError::WouldBlock);
}
dst.write(&buf[..n]).map_err(|_| AxError::Io)
}))
}

fn write(&self, _src: &mut crate::file::IoSrc) -> AxResult<usize> {
Expand Down Expand Up @@ -157,6 +188,15 @@ impl FileLike for PerfEvent {
Some(anchor),
))
}

fn nonblocking(&self) -> bool {
self.nonblocking.load(Ordering::Acquire)
}

fn set_nonblocking(&self, on: bool) -> AxResult {
self.nonblocking.store(on, Ordering::Release);
Ok(())
}
}

/// `perf_event_open(2)` syscall entry. Copies the user `perf_event_attr` in
Expand Down
8 changes: 4 additions & 4 deletions os/StarryOS/kernel/src/perf/uprobe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use ax_errno::{AxError, AxResult};
use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig};
use kprobe::ProbeBuilder;

use super::kprobe::{ProbePerfEvent, ProbeTy};
use super::kprobe::{PROBE_CONFIG_ENTRY, PROBE_CONFIG_RETURN, ProbePerfEvent, ProbeTy};
use crate::{
kprobe::KprobeAuxiliary,
task::{AsThread, get_task},
Expand Down Expand Up @@ -74,13 +74,13 @@ fn perf_probe_arg_to_uprobe_builder(
/// Build a uprobe perf event from `perf_event_open` args.
pub fn perf_event_open_uprobe(args: PerfProbeArgs) -> AxResult<ProbePerfEvent> {
let probe = match args.config {
PerfProbeConfig::Raw(0) => {
PerfProbeConfig::Raw(PROBE_CONFIG_ENTRY) => {
let builder = perf_probe_arg_to_uprobe_builder(&args)?;
ProbeTy::Uprobe(crate::uprobe::register_uprobe(builder))
}
PerfProbeConfig::Raw(1) => {
PerfProbeConfig::Raw(PROBE_CONFIG_RETURN) => {
// uretprobe — not implemented for user space yet.
warn!("uprobe: uretprobe (config=1) is not yet supported");
warn!("uprobe: uretprobe is not yet supported");
return Err(AxError::Unsupported);
}
other => {
Expand Down
3 changes: 2 additions & 1 deletion os/StarryOS/kernel/src/pseudofs/dev/card1.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,8 @@ impl DeviceOps for Card1 {
}

_ => {
panic!("card1: unsupported ioctl nr {nr:#x}");
warn!("card1: unsupported ioctl nr {nr:#x}");
return Err(VfsError::OperationNotSupported);
}
}
copy_to_user(arg as _, stack_data.as_mut_ptr(), out_size)?;
Expand Down
Loading
Loading