Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 112 additions & 2 deletions os/StarryOS/kernel/src/perf/bpf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr};
use ax_task::IrqNotify;
use axpoll::{IoEvents, PollSet, Pollable};
use kbpf_basic::{
linux_bpf::perf_event_sample_format,
linux_bpf::{perf_event_header, perf_event_mmap_page, perf_event_sample_format},
perf::{PerfProbeArgs, bpf::BpfPerfEvent},
};
use kprobe::PtRegs;
Expand All @@ -37,6 +37,11 @@ use crate::{
file::FileLike,
};

/// Number of 4K pages reserved for x86_64 BPF JIT executable memory.
/// Each JIT-compiled eBPF program fits within this space; the allocation
/// is sized generously (16 KiB) since programs are typically < 1 page.
const BPF_JIT_MEM_PAGES: usize = 4;

/// Wraps `kbpf_basic::perf::bpf::BpfPerfEvent` with kernel state: a poll
/// set so readers can wait for new records, and a weak handle to the
/// backing pages produced by `device_mmap` (Some after the first
Expand Down Expand Up @@ -72,6 +77,9 @@ pub struct BpfPerfEventWrapper {
/// mapping still exists. See the type-level docs for the ownership
/// rationale.
pages: Option<Weak<GlobalPage>>,
/// Kernel virtual address of the ringbuf start, saved during
/// `device_mmap` for `read(2)` access.
mmap_kvirt: Option<usize>,
}

impl BpfPerfEventWrapper {
Expand All @@ -87,6 +95,7 @@ impl BpfPerfEventWrapper {
poll_notify,
poll_alive,
pages: None,
mmap_kvirt: None,
}
}

Expand All @@ -113,6 +122,77 @@ impl BpfPerfEventWrapper {
}
Ok(())
}

/// Read one ringbuf record into `dst`, returning the number of bytes
/// copied (including the `perf_event_header`). Returns `Ok(0)` when no
/// data is available. Safety: the ringbuf pages are valid as long as
/// the kernel virtual address was saved during `device_mmap` and the
/// strong page ref still exists (checked via `is_mapped`).
pub(crate) fn try_read_record(&self, dst: &mut [u8]) -> AxResult<usize> {
let kvirt = self.mmap_kvirt.ok_or(AxError::NotConnected)?;
if !self.is_mapped() {
return Ok(0);
}

// SAFETY: the ringbuf pages are pinned by the VMA and zero-
// initialised; the `perf_event_mmap_page` at offset 0 was
// initialised by `BpfPerfEvent::do_mmap` during `device_mmap`.
let mmap = unsafe { &*(kvirt as *const perf_event_mmap_page) };
if mmap.data_tail == mmap.data_head {
return Ok(0);
}

let data_region_size = mmap.data_size as usize;
let tail = (mmap.data_tail as usize) % data_region_size;

// SAFETY: the data region starts at PAGE_SIZE offset into the
// ringbuf allocation; `data_size` was set by kbpf-basic during
// `do_mmap` and bounds the accessible region.
let data_region = unsafe {
core::slice::from_raw_parts((kvirt + PAGE_SIZE_4K) as *const u8, data_region_size)
};

let hdr_size = core::mem::size_of::<perf_event_header>();
if hdr_size > dst.len() {
return Err(AxError::InvalidInput);
}

// Read the perf_event_header (handling ring-wrap).
let hdr_end = wrapping_add(tail, hdr_size, data_region_size);
copy_ring(data_region, tail, hdr_end, &mut dst[..hdr_size]);

// SAFETY: the header bytes are a valid `perf_event_header` written
// by kbpf-basic's `RingPage::write_sample`.
let header = unsafe { &*(dst.as_ptr() as *const perf_event_header) };
let record_size = header.size as usize;

if record_size < hdr_size || record_size > dst.len() {
warn!(
"eBPF perf: bad record size {record_size} (hdr_size={hdr_size}, buf_len={})",
dst.len()
);
return Err(AxError::InvalidInput);
}

// Copy the full record (header already copied at offset 0).
if record_size > hdr_size {
let payload_start = wrapping_add(tail, hdr_size, data_region_size);
let payload_end = wrapping_add(tail, record_size, data_region_size);
copy_ring(
data_region,
payload_start,
payload_end,
&mut dst[hdr_size..record_size],
);
}

// Advance data_tail past this record.
// SAFETY: the mmap page is writable by the kernel.
let mmap_mut = unsafe { &mut *(kvirt as *mut perf_event_mmap_page) };
mmap_mut.data_tail += record_size as u64;

Ok(record_size)
}
}

impl Drop for BpfPerfEventWrapper {
Expand Down Expand Up @@ -146,6 +226,35 @@ impl Debug for BpfPerfEventWrapper {
}
}

/// Compute `(base + offset) % modulus`. The modulus is always the ring
/// data-region size so the result fits in a single wrapping-modulo round.
#[inline]
fn wrapping_add(base: usize, offset: usize, modulus: usize) -> usize {
debug_assert!(modulus > 0);
let sum = base + offset;
if sum < modulus { sum } else { sum % modulus }
}

/// Copy bytes from a ring buffer (`src`) into a linear buffer (`dst`).
/// The ring extent is `[from, to)` with `from` and `to` modulo-wrapped into
/// the ring size. Handles the zero or one wrap case.
fn copy_ring(src: &[u8], from: usize, to: usize, dst: &mut [u8]) {
let ring_size = src.len();
let len = to.wrapping_sub(from) % ring_size;
if len == 0 {
return;
}
debug_assert!(len <= dst.len(), "copy_ring: dst too small");
let end = from + len;
if end <= ring_size {
dst[..len].copy_from_slice(&src[from..end]);
} else {
let first = ring_size - from;
dst[..first].copy_from_slice(&src[from..]);
dst[first..len].copy_from_slice(&src[..(end % ring_size)]);
}
}

impl PerfEventOps for BpfPerfEventWrapper {
fn enable(&mut self) -> AxResult<()> {
self.inner.enable().into_ax_result()?;
Expand Down Expand Up @@ -196,6 +305,7 @@ impl PerfEventOps for BpfPerfEventWrapper {
// (see the type-level docs). Without the anchor the pages would free
// under a live mapping.
self.pages = Some(Arc::downgrade(&pages));
self.mmap_kvirt = Some(kvirt.as_usize());
let anchor: Arc<dyn Any + Send + Sync> = pages;
Ok((paddr, anchor))
}
Expand Down Expand Up @@ -283,7 +393,7 @@ impl OwnedEbpfVm {
#[cfg(target_arch = "x86_64")]
{
// TODO: calculate a more precise size.
let mut jit_exec_memory = BPFJitMemory::new(4)?;
let mut jit_exec_memory = BPFJitMemory::new(BPF_JIT_MEM_PAGES)?;
// SAFETY: `jit_exec_memory` is moved into the returned
// `OwnedEbpfVm` after `vm`; field drop order guarantees `vm` is
// destroyed before the executable mapping is unmapped.
Expand Down
35 changes: 21 additions & 14 deletions os/StarryOS/kernel/src/perf/kprobe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ use axpoll::Pollable;
use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig};
use kprobe::{CallBackFunc, KretprobeBuilder, ProbeBuilder, PtRegs};

/// Config value for entry probes (kprobe/uprobe), per Linux PERF_TYPE_PROBE ABI.
pub const PROBE_CONFIG_ENTRY: u64 = 0;
/// Config value for return probes (kretprobe/uretprobe), per Linux PERF_TYPE_PROBE ABI.
pub const PROBE_CONFIG_RETURN: u64 = 1;
/// Maximum number of concurrently active kretprobe instances, matching
/// Linux's `max(10, 2*NR_CPUS)` default for single-CPU configurations.
const KRETPROBE_MAX_ACTIVE: u32 = 10;

use crate::{
file::FileLike,
kprobe::{
Expand Down Expand Up @@ -182,25 +190,24 @@ fn perf_probe_arg_to_kretprobe_builder(
) -> AxResult<KretprobeBuilder<KernelRawMutex>> {
let symbol = &args.name;
let addr = lookup_symbol_addr(symbol)?;
Ok(KretprobeBuilder::<KernelRawMutex>::new(10)
.with_symbol(symbol.clone())
.with_symbol_addr(addr))
Ok(
KretprobeBuilder::<KernelRawMutex>::new(KRETPROBE_MAX_ACTIVE)
.with_symbol(symbol.clone())
.with_symbol_addr(addr),
)
}

/// Build a `ProbePerfEvent` for a `PERF_TYPE_KPROBE` perf_event_open call.
/// Config 0 = kprobe; config 1 = kretprobe (per kbpf-basic convention).
/// Config `PROBE_CONFIG_ENTRY` (0) = kprobe; `PROBE_CONFIG_RETURN` (1) = kretprobe.
pub fn perf_event_open_kprobe(args: PerfProbeArgs) -> AxResult<ProbePerfEvent> {
let probe = match args.config {
PerfProbeConfig::Raw(val) => {
if val == 0 {
let builder = perf_probe_arg_to_kprobe_builder(&args)?;
ProbeTy::Kprobe(register_kprobe(builder))
} else if val == 1 {
let builder = perf_probe_arg_to_kretprobe_builder(&args)?;
ProbeTy::Kretprobe(register_kretprobe(builder))
} else {
return Err(AxError::InvalidInput);
}
PerfProbeConfig::Raw(PROBE_CONFIG_ENTRY) => {
let builder = perf_probe_arg_to_kprobe_builder(&args)?;
ProbeTy::Kprobe(register_kprobe(builder))
}
PerfProbeConfig::Raw(PROBE_CONFIG_RETURN) => {
let builder = perf_probe_arg_to_kretprobe_builder(&args)?;
ProbeTy::Kretprobe(register_kretprobe(builder))
}
_ => return Err(AxError::InvalidInput),
};
Expand Down
28 changes: 25 additions & 3 deletions os/StarryOS/kernel/src/perf/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ use ax_kspin::{SpinNoPreempt, SpinNoPreemptGuard};
use ax_lazyinit::LazyInit;
use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr, PhysAddrRange, VirtAddr, VirtAddrRange};
use ax_runtime::hal::paging::MappingFlags;
use axpoll::Pollable;
use ax_task::future::{block_on, poll_io};
use axpoll::{IoEvents, Pollable};
pub use bpf::BpfPerfEventWrapper;
use hashbrown::HashMap;
use kbpf_basic::{
Expand Down Expand Up @@ -105,9 +106,30 @@ impl Pollable for PerfEvent {
}
}

/// Stack buffer size for reading a single perf ringbuf record. The
/// ringbuf data region minimum is one page, and a single perf record
/// always fits within one data-region page.
const PERF_READ_BUF_SIZE: usize = PAGE_SIZE_4K;

impl FileLike for PerfEvent {
fn read(&self, _dst: &mut crate::file::IoDst) -> AxResult<usize> {
Err(AxError::Unsupported)
fn read(&self, dst: &mut crate::file::IoDst) -> AxResult<usize> {
block_on(poll_io(self, IoEvents::IN, self.nonblocking(), || {
let mut event = self.event.lock();
let bpf_event = event
.as_any_mut()
.downcast_mut::<BpfPerfEventWrapper>()
.ok_or(AxError::Unsupported)?;
let mut buf = [0u8; PERF_READ_BUF_SIZE];
let n = bpf_event.try_read_record(&mut buf)?;
// Release the lock before writing to dst (I/O outside the spin
// lock) and before returning WouldBlock (the lock must not be
// held across the poll_io reschedule).
drop(event);
if n == 0 {
return Err(AxError::WouldBlock);
}
dst.write(&buf[..n]).map_err(|_| AxError::Io)
}))
}

fn write(&self, _src: &mut crate::file::IoSrc) -> AxResult<usize> {
Expand Down
8 changes: 4 additions & 4 deletions os/StarryOS/kernel/src/perf/uprobe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use ax_errno::{AxError, AxResult};
use kbpf_basic::perf::{PerfProbeArgs, PerfProbeConfig};
use kprobe::ProbeBuilder;

use super::kprobe::{ProbePerfEvent, ProbeTy};
use super::kprobe::{PROBE_CONFIG_ENTRY, PROBE_CONFIG_RETURN, ProbePerfEvent, ProbeTy};
use crate::{
kprobe::KprobeAuxiliary,
task::{AsThread, get_task},
Expand Down Expand Up @@ -74,13 +74,13 @@ fn perf_probe_arg_to_uprobe_builder(
/// Build a uprobe perf event from `perf_event_open` args.
pub fn perf_event_open_uprobe(args: PerfProbeArgs) -> AxResult<ProbePerfEvent> {
let probe = match args.config {
PerfProbeConfig::Raw(0) => {
PerfProbeConfig::Raw(PROBE_CONFIG_ENTRY) => {
let builder = perf_probe_arg_to_uprobe_builder(&args)?;
ProbeTy::Uprobe(crate::uprobe::register_uprobe(builder))
}
PerfProbeConfig::Raw(1) => {
PerfProbeConfig::Raw(PROBE_CONFIG_RETURN) => {
// uretprobe — not implemented for user space yet.
warn!("uprobe: uretprobe (config=1) is not yet supported");
warn!("uprobe: uretprobe is not yet supported");
return Err(AxError::Unsupported);
}
other => {
Expand Down
Loading