Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions os/StarryOS/kernel/src/ebpf/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ use crate::{
/// without re-running the kbpf-basic init on every call.
pub static BPF_HELPER_FUN_SET: LazyInit<BTreeMap<u32, RawBPFHelperFn>> = LazyInit::new();

/// BPF helper ID for `bpf_probe_read` (legacy, address-space-agnostic).
const BPF_FUNC_PROBE_READ: u32 = 4;
/// BPF helper ID for `bpf_probe_read_kernel`. kbpf-basic only registers
/// `bpf_probe_read` so we alias 113 onto the same raw helper.
const BPF_FUNC_PROBE_READ_KERNEL: u32 = 113;

/// Initialize the BPF subsystem: build the helper-function table from
/// `kbpf-basic`. Must be called before `sys_bpf(BPF_PROG_LOAD)` so loaded
/// programs can resolve the helper ids referenced in their instructions.
Expand All @@ -61,8 +67,8 @@ pub fn init_ebpf() {
// tracepoint program uses to pull fields out of its sample buffer.
// kbpf-basic only registers the legacy `bpf_probe_read` (id 4), whose raw
// reader is address-space-agnostic in this VM, so alias 113 onto it.
if let Some(&probe_read) = set.get(&4) {
set.entry(113).or_insert(probe_read);
if let Some(&probe_read) = set.get(&BPF_FUNC_PROBE_READ) {
set.entry(BPF_FUNC_PROBE_READ_KERNEL).or_insert(probe_read);
}
BPF_HELPER_FUN_SET.init_once(set);
}
Expand Down
2 changes: 1 addition & 1 deletion os/StarryOS/kernel/src/mm/loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -520,7 +520,7 @@ impl ElfCacheEntry {
fn load(loc: Location) -> AxResult<Result<Self, Vec<u8>>> {
let cache = CachedFile::get_or_create(loc)?;

let mut data = vec![0; 4096];
let mut data = vec![0; PAGE_SIZE_4K];
let read = cache.read_at(&mut data[..], 0)?;
data.truncate(read);
match ElfCacheEntry::try_new_or_recover::<AxError>(cache.clone(), data, |data| {
Expand Down
132 changes: 124 additions & 8 deletions os/StarryOS/kernel/src/perf/bpf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr};
use ax_task::IrqNotify;
use axpoll::{IoEvents, PollSet, Pollable};
use kbpf_basic::{
linux_bpf::perf_event_sample_format,
linux_bpf::{perf_event_header, perf_event_mmap_page, perf_event_sample_format},
perf::{PerfProbeArgs, bpf::BpfPerfEvent},
};
use kprobe::PtRegs;
Expand All @@ -37,6 +37,11 @@ use crate::{
file::FileLike,
};

/// Number of 4K pages reserved for x86_64 BPF JIT executable memory.
/// Each JIT-compiled eBPF program fits within this space; the allocation
/// is sized generously (16 KiB) since programs are typically < 1 page.
const BPF_JIT_MEM_PAGES: usize = 4;

/// Wraps `kbpf_basic::perf::bpf::BpfPerfEvent` with kernel state: a poll
/// set so readers can wait for new records, and a weak handle to the
/// backing pages produced by `device_mmap` (Some after the first
Expand Down Expand Up @@ -72,6 +77,9 @@ pub struct BpfPerfEventWrapper {
/// mapping still exists. See the type-level docs for the ownership
/// rationale.
pages: Option<Weak<GlobalPage>>,
/// Kernel virtual address of the ringbuf start, saved during
/// `device_mmap` for `read(2)` access.
mmap_kvirt: Option<usize>,
}

impl BpfPerfEventWrapper {
Expand All @@ -87,6 +95,7 @@ impl BpfPerfEventWrapper {
poll_notify,
poll_alive,
pages: None,
mmap_kvirt: None,
}
}

Expand All @@ -113,6 +122,77 @@ impl BpfPerfEventWrapper {
}
Ok(())
}

/// Read one ringbuf record into `dst`, returning the number of bytes
/// copied (including the `perf_event_header`). Returns `Ok(0)` when no
/// data is available. Safety: the ringbuf pages are valid as long as
/// the kernel virtual address was saved during `device_mmap` and the
/// strong page ref still exists (checked via `is_mapped`).
pub(crate) fn try_read_record(&self, dst: &mut [u8]) -> AxResult<usize> {
let kvirt = self.mmap_kvirt.ok_or(AxError::NotConnected)?;
if !self.is_mapped() {
return Ok(0);
}

// SAFETY: the ringbuf pages are pinned by the VMA and zero-
// initialised; the `perf_event_mmap_page` at offset 0 was
// initialised by `BpfPerfEvent::do_mmap` during `device_mmap`.
let mmap = unsafe { &*(kvirt as *const perf_event_mmap_page) };
if mmap.data_tail == mmap.data_head {
return Ok(0);
}

let data_region_size = mmap.data_size as usize;
let tail = (mmap.data_tail as usize) % data_region_size;

// SAFETY: the data region starts at PAGE_SIZE offset into the
// ringbuf allocation; `data_size` was set by kbpf-basic during
// `do_mmap` and bounds the accessible region.
let data_region = unsafe {
core::slice::from_raw_parts((kvirt + PAGE_SIZE_4K) as *const u8, data_region_size)
};

let hdr_size = core::mem::size_of::<perf_event_header>();
if hdr_size > dst.len() {
return Err(AxError::InvalidInput);
}

// Read the perf_event_header (handling ring-wrap).
let hdr_end = wrapping_add(tail, hdr_size, data_region_size);
copy_ring(data_region, tail, hdr_end, &mut dst[..hdr_size]);

// SAFETY: the header bytes are a valid `perf_event_header` written
// by kbpf-basic's `RingPage::write_sample`.
let header = unsafe { &*(dst.as_ptr() as *const perf_event_header) };
let record_size = header.size as usize;

if record_size < hdr_size || record_size > dst.len() {
warn!(
"eBPF perf: bad record size {record_size} (hdr_size={hdr_size}, buf_len={})",
dst.len()
);
return Err(AxError::InvalidInput);
}

// Copy the full record (header already copied at offset 0).
if record_size > hdr_size {
let payload_start = wrapping_add(tail, hdr_size, data_region_size);
let payload_end = wrapping_add(tail, record_size, data_region_size);
copy_ring(
data_region,
payload_start,
payload_end,
&mut dst[hdr_size..record_size],
);
}

// Advance data_tail past this record.
// SAFETY: the mmap page is writable by the kernel.
let mmap_mut = unsafe { &mut *(kvirt as *mut perf_event_mmap_page) };
mmap_mut.data_tail += record_size as u64;

Ok(record_size)
}
}

impl Drop for BpfPerfEventWrapper {
Expand Down Expand Up @@ -146,6 +226,35 @@ impl Debug for BpfPerfEventWrapper {
}
}

/// Compute `(base + offset) % modulus`. The modulus is always the ring
/// data-region size so the result fits in a single wrapping-modulo round.
#[inline]
fn wrapping_add(base: usize, offset: usize, modulus: usize) -> usize {
debug_assert!(modulus > 0);
let sum = base + offset;
if sum < modulus { sum } else { sum % modulus }
}

/// Copy bytes from a ring buffer (`src`) into a linear buffer (`dst`).
/// The ring extent is `[from, to)` with `from` and `to` modulo-wrapped into
/// the ring size. Handles the zero or one wrap case.
fn copy_ring(src: &[u8], from: usize, to: usize, dst: &mut [u8]) {
let ring_size = src.len();
let len = to.wrapping_sub(from) % ring_size;
if len == 0 {
return;
}
debug_assert!(len <= dst.len(), "copy_ring: dst too small");
let end = from + len;
if end <= ring_size {
dst[..len].copy_from_slice(&src[from..end]);
} else {
let first = ring_size - from;
dst[..first].copy_from_slice(&src[from..]);
dst[first..len].copy_from_slice(&src[..(end % ring_size)]);
}
}

impl PerfEventOps for BpfPerfEventWrapper {
fn enable(&mut self) -> AxResult<()> {
self.inner.enable().into_ax_result()?;
Expand Down Expand Up @@ -196,6 +305,7 @@ impl PerfEventOps for BpfPerfEventWrapper {
// (see the type-level docs). Without the anchor the pages would free
// under a live mapping.
self.pages = Some(Arc::downgrade(&pages));
self.mmap_kvirt = Some(kvirt.as_usize());
let anchor: Arc<dyn Any + Send + Sync> = pages;
Ok((paddr, anchor))
}
Expand Down Expand Up @@ -273,17 +383,23 @@ impl OwnedEbpfVm {
let _ = vm.register_helper(*key, *value);
}
}
// TODO: not all of the address space is accessible to a BPF program;
// allowing the full `0..u64::MAX` range disables rbpf's bounds check
// and lets a buggy/hostile program read arbitrary kernel memory via
// direct loads. Narrow this to the legitimately-reachable context /
// map / stack ranges once kbpf-basic exposes the per-program bounds.
vm.register_allowed_memory(0..u64::MAX);
// bpftrace / libbpf / aya observability programs access kernel
// memory exclusively through registered helper functions (e.g.
// bpf_probe_read, bpf_map_lookup_elem), never via direct LD_ABS
// / LD_IND / LD_DW loads. We intentionally do NOT call
// `register_allowed_memory` so rbpf's bounds check rejects any
// direct memory access, preventing a buggy or hostile program
// from reading arbitrary kernel memory.
//
// If a future program legitimately needs direct loads (e.g. a
// raw socket filter that touches packet data), register its
// specific buffer range here rather than re-opening the full
// address space.

#[cfg(target_arch = "x86_64")]
{
// TODO: calculate a more precise size.
let mut jit_exec_memory = BPFJitMemory::new(4)?;
let mut jit_exec_memory = BPFJitMemory::new(BPF_JIT_MEM_PAGES)?;
// SAFETY: `jit_exec_memory` is moved into the returned
// `OwnedEbpfVm` after `vm`; field drop order guarantees `vm` is
// destroyed before the executable mapping is unmapped.
Expand Down
35 changes: 21 additions & 14 deletions os/StarryOS/kernel/src/perf/kprobe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ use axpoll::Pollable;
use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig};
use kprobe::{CallBackFunc, KretprobeBuilder, ProbeBuilder, PtRegs};

/// Config value for entry probes (kprobe/uprobe), per Linux PERF_TYPE_PROBE ABI.
pub const PROBE_CONFIG_ENTRY: u64 = 0;
/// Config value for return probes (kretprobe/uretprobe), per Linux PERF_TYPE_PROBE ABI.
pub const PROBE_CONFIG_RETURN: u64 = 1;
/// Maximum number of concurrently active kretprobe instances, matching
/// Linux's `max(10, 2*NR_CPUS)` default for single-CPU configurations.
const KRETPROBE_MAX_ACTIVE: u32 = 10;

use crate::{
file::FileLike,
kprobe::{
Expand Down Expand Up @@ -182,25 +190,24 @@ fn perf_probe_arg_to_kretprobe_builder(
) -> AxResult<KretprobeBuilder<KernelRawMutex>> {
let symbol = &args.name;
let addr = lookup_symbol_addr(symbol)?;
Ok(KretprobeBuilder::<KernelRawMutex>::new(10)
.with_symbol(symbol.clone())
.with_symbol_addr(addr))
Ok(
KretprobeBuilder::<KernelRawMutex>::new(KRETPROBE_MAX_ACTIVE)
.with_symbol(symbol.clone())
.with_symbol_addr(addr),
)
}

/// Build a `ProbePerfEvent` for a `PERF_TYPE_KPROBE` perf_event_open call.
/// Config 0 = kprobe; config 1 = kretprobe (per kbpf-basic convention).
/// Config `PROBE_CONFIG_ENTRY` (0) = kprobe; `PROBE_CONFIG_RETURN` (1) = kretprobe.
pub fn perf_event_open_kprobe(args: PerfProbeArgs) -> AxResult<ProbePerfEvent> {
let probe = match args.config {
PerfProbeConfig::Raw(val) => {
if val == 0 {
let builder = perf_probe_arg_to_kprobe_builder(&args)?;
ProbeTy::Kprobe(register_kprobe(builder))
} else if val == 1 {
let builder = perf_probe_arg_to_kretprobe_builder(&args)?;
ProbeTy::Kretprobe(register_kretprobe(builder))
} else {
return Err(AxError::InvalidInput);
}
PerfProbeConfig::Raw(PROBE_CONFIG_ENTRY) => {
let builder = perf_probe_arg_to_kprobe_builder(&args)?;
ProbeTy::Kprobe(register_kprobe(builder))
}
PerfProbeConfig::Raw(PROBE_CONFIG_RETURN) => {
let builder = perf_probe_arg_to_kretprobe_builder(&args)?;
ProbeTy::Kretprobe(register_kretprobe(builder))
}
_ => return Err(AxError::InvalidInput),
};
Expand Down
48 changes: 44 additions & 4 deletions os/StarryOS/kernel/src/perf/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,21 @@ pub mod tracepoint;
pub mod uprobe;

use alloc::{borrow::Cow, boxed::Box, sync::Arc, vec};
use core::{any::Any, ffi::c_void, fmt::Debug};
use core::{
any::Any,
ffi::c_void,
fmt::Debug,
sync::atomic::{AtomicBool, Ordering},
};

use ax_errno::{AxError, AxResult};
use ax_io::Read;
use ax_kspin::{SpinNoPreempt, SpinNoPreemptGuard};
use ax_lazyinit::LazyInit;
use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr, PhysAddrRange, VirtAddr, VirtAddrRange};
use ax_runtime::hal::paging::MappingFlags;
use axpoll::Pollable;
use ax_task::future::{block_on, poll_io};
use axpoll::{IoEvents, Pollable};
pub use bpf::BpfPerfEventWrapper;
use hashbrown::HashMap;
use kbpf_basic::{
Expand Down Expand Up @@ -73,6 +79,9 @@ pub trait PerfEventOps: Pollable + Send + Sync + Debug {
/// `Box<dyn PerfEventOps>` so the inner implementation can stay generic.
pub struct PerfEvent {
event: SpinNoPreempt<Box<dyn PerfEventOps>>,
/// O_NONBLOCK flag. When true, `read(2)` returns EAGAIN instead of
/// blocking on an empty ringbuf.
nonblocking: AtomicBool,
}

impl Debug for PerfEvent {
Expand All @@ -86,6 +95,7 @@ impl PerfEvent {
pub fn new(event: Box<dyn PerfEventOps>) -> Self {
PerfEvent {
event: SpinNoPreempt::new(event),
nonblocking: AtomicBool::new(false),
}
}

Expand All @@ -105,9 +115,30 @@ impl Pollable for PerfEvent {
}
}

/// Stack buffer size for reading a single perf ringbuf record. The
/// ringbuf data region minimum is one page, and a single perf record
/// always fits within one data-region page.
const PERF_READ_BUF_SIZE: usize = PAGE_SIZE_4K;

impl FileLike for PerfEvent {
fn read(&self, _dst: &mut crate::file::IoDst) -> AxResult<usize> {
Err(AxError::Unsupported)
fn read(&self, dst: &mut crate::file::IoDst) -> AxResult<usize> {
block_on(poll_io(self, IoEvents::IN, self.nonblocking(), || {
let mut event = self.event.lock();
let bpf_event = event
.as_any_mut()
.downcast_mut::<BpfPerfEventWrapper>()
.ok_or(AxError::Unsupported)?;
let mut buf = [0u8; PERF_READ_BUF_SIZE];
let n = bpf_event.try_read_record(&mut buf)?;
// Release the lock before writing to dst (I/O outside the spin
// lock) and before returning WouldBlock (the lock must not be
// held across the poll_io reschedule).
drop(event);
if n == 0 {
return Err(AxError::WouldBlock);
}
dst.write(&buf[..n]).map_err(|_| AxError::Io)
}))
}

fn write(&self, _src: &mut crate::file::IoSrc) -> AxResult<usize> {
Expand Down Expand Up @@ -157,6 +188,15 @@ impl FileLike for PerfEvent {
Some(anchor),
))
}

fn nonblocking(&self) -> bool {
self.nonblocking.load(Ordering::Acquire)
}

fn set_nonblocking(&self, on: bool) -> AxResult {
self.nonblocking.store(on, Ordering::Release);
Ok(())
}
}

/// `perf_event_open(2)` syscall entry. Copies the user `perf_event_attr` in
Expand Down
8 changes: 4 additions & 4 deletions os/StarryOS/kernel/src/perf/uprobe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use ax_errno::{AxError, AxResult};
use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig};
use kprobe::ProbeBuilder;

use super::kprobe::{ProbePerfEvent, ProbeTy};
use super::kprobe::{PROBE_CONFIG_ENTRY, PROBE_CONFIG_RETURN, ProbePerfEvent, ProbeTy};
use crate::{
kprobe::KprobeAuxiliary,
task::{AsThread, get_task},
Expand Down Expand Up @@ -74,13 +74,13 @@ fn perf_probe_arg_to_uprobe_builder(
/// Build a uprobe perf event from `perf_event_open` args.
pub fn perf_event_open_uprobe(args: PerfProbeArgs) -> AxResult<ProbePerfEvent> {
let probe = match args.config {
PerfProbeConfig::Raw(0) => {
PerfProbeConfig::Raw(PROBE_CONFIG_ENTRY) => {
let builder = perf_probe_arg_to_uprobe_builder(&args)?;
ProbeTy::Uprobe(crate::uprobe::register_uprobe(builder))
}
PerfProbeConfig::Raw(1) => {
PerfProbeConfig::Raw(PROBE_CONFIG_RETURN) => {
// uretprobe — not implemented for user space yet.
warn!("uprobe: uretprobe (config=1) is not yet supported");
warn!("uprobe: uretprobe is not yet supported");
return Err(AxError::Unsupported);
}
other => {
Expand Down
Loading
Loading