diff --git a/components/axbacktrace/src/lib.rs b/components/axbacktrace/src/lib.rs index a66dd4d60f..34dc63c1cb 100644 --- a/components/axbacktrace/src/lib.rs +++ b/components/axbacktrace/src/lib.rs @@ -49,6 +49,121 @@ pub fn init(ip_range: Range, fp_range: Range) { dwarf::init(); } +/// The kernel instruction (text) range passed to [`init`] — i.e. `[_stext, _etext)`. +/// +/// `None` before [`init`]. Reading a [`spin::Once`] is a lock-free acquire load +/// with no spinning, so this is safe to call from hard-IRQ context (e.g. the PMU +/// sampling call-graph unwinder, which reuses these ranges as the walker's IP +/// filter). +pub fn ip_range() -> Option> { + IP_RANGE.get().cloned() +} + +/// The kernel frame-pointer range passed to [`init`] — i.e. the kernel address +/// space `[kernel_space_start, kernel_space_end)`. +/// +/// `None` before [`init`]. Lock-free and IRQ-safe (see [`ip_range`]). +pub fn fp_range() -> Option> { + FP_RANGE.get().cloned() +} + +/// Allocation-free frame-pointer stack walk into a caller-supplied buffer. +/// +/// Writes the leaf `pc` as `out[0]`, then walks the AAPCS64 / frame-record chain +/// (`[fp]` = caller fp, `[fp + 8]` = saved return address) appending each return +/// address that lies in `ip_range`, and returns the number of `u64` entries +/// written (`>= 1`). Unlike [`unwind_stack`]/`unwind_core`, this allocates +/// nothing, terminates at `out.len()`, and reads memory through the injected +/// `read` closure instead of dereferencing directly — so a caller in hard-IRQ +/// context can supply a fault-safe reader (a direct deref for always-mapped +/// kernel frames, a no-fault page-table walk for user frames). +/// +/// The guard set mirrors [`unwind_core`]: `fp` must stay within `fp_range` and be +/// word-aligned, must advance monotonically (a non-advancing or backward `fp` +/// ends the walk), and a jump of 8 MiB or more between adjacent frames ends it. +/// A return address outside `ip_range` is skipped (not recorded) but the walk +/// continues, since a single bad IP does not imply the whole chain is corrupt; +/// a bounded *total*-iteration budget (a small multiple of `out.len()`) caps that +/// skip path so a chain of all-skipped frames cannot spin. `read(va)` returns the +/// machine word at `va`, or `None` if it cannot be read safely; a failed read +/// ends the walk. These guards bound the walk so a corrupted chain can neither +/// loop nor run past the depth cap. +pub fn walk_fp( + pc: usize, + mut fp: usize, + ip_range: &Range, + fp_range: &Range, + read: impl Fn(usize) -> Option, + out: &mut [u64], +) -> usize { + /// AAPCS64 frame record: the saved return address sits one word above the + /// saved caller frame pointer. + const FP_TO_LR_OFFSET: usize = core::mem::size_of::(); + /// Largest plausible gap between adjacent frame pointers; a larger jump means + /// the chain is corrupt (mirrors `unwind_core`'s 8 MiB guard). + const MAX_FRAME_GAP: usize = 8 * 1024 * 1024; + /// Total-iteration budget as a multiple of the recorded-frame cap. A frame + /// whose return address is outside `ip_range` is skipped without consuming a + /// recorded slot, so `n < out.len()` alone does not bound the loop; this caps + /// *total* iterations so a crafted all-skip chain (every `[fp+8]` out of range) + /// cannot spin the walk — which matters when `read` does real work per step + /// (two no-fault page-table walks in the user reader). + const MAX_STEP_FACTOR: usize = 4; + + if out.is_empty() { + return 0; + } + out[0] = pc as u64; + let mut n = 1; + let mut steps = 0usize; + let max_steps = out.len().saturating_mul(MAX_STEP_FACTOR); + + while n < out.len() && steps < max_steps { + steps += 1; + // The whole frame record ([fp] = caller fp, [fp + 8] = saved LR) must sit + // inside the stack range and `fp` be word-aligned. Bounding `fp + 8` (not + // just `fp`) keeps the saved-LR read below from touching one word past + // `fp_range.end` — which matters for a direct-deref reader. + let record_end = fp.wrapping_add(2 * FP_TO_LR_OFFSET); + if !fp_range.contains(&fp) + || record_end > fp_range.end + || record_end < fp + || !fp.is_multiple_of(core::mem::align_of::()) + { + break; + } + let Some(caller_fp) = read(fp) else { break }; + let Some(lr) = read(fp.wrapping_add(FP_TO_LR_OFFSET)) else { + break; + }; + + // The top of the chain (fp == 0) or a non-advancing / backward / wildly + // distant caller ends the walk. Checked here so it bounds BOTH the skip and + // record paths — a small-gap all-skip chain must still be step-capped above. + if caller_fp == 0 { + // Record a final in-range leaf return address before stopping. + if ip_range.contains(&lr) { + out[n] = lr as u64; + n += 1; + } + break; + } + if caller_fp <= fp || caller_fp - fp >= MAX_FRAME_GAP { + break; + } + + // Skip a return address outside the code range but keep unwinding: one + // bad IP does not necessarily mean the FP chain itself is broken. + if ip_range.contains(&lr) { + out[n] = lr as u64; + n += 1; + } + fp = caller_fp; + } + + n +} + /// Represents a single stack frame in the unwound stack. #[repr(C)] #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] @@ -454,11 +569,28 @@ impl fmt::Debug for Backtrace { mod tests { use alloc::{boxed::Box, format, vec::Vec}; + extern crate std; + + use std::sync::{Mutex, MutexGuard}; + use super::*; - fn init_for_tests() { + /// Serializes tests that read or mutate the process-global `MAX_DEPTH` + /// (`set_max_depth`/`max_depth`). Without it, a test that lowers the depth + /// (e.g. `stress_deep_chain_truncation` -> 16) races the depth its neighbours + /// assume (`init_for_tests` -> 32), which flakes under `cargo test`'s parallel + /// harness. Every depth-sensitive test holds this for its whole body via the + /// guard returned from `init_for_tests`. + static DEPTH_SERIAL: Mutex<()> = Mutex::new(()); + + #[must_use] + fn init_for_tests() -> MutexGuard<'static, ()> { + // Tolerate poisoning: a panicking (failing) test must not cascade-poison + // every later test into a spurious failure — we only need mutual exclusion. + let guard = DEPTH_SERIAL.lock().unwrap_or_else(|e| e.into_inner()); init(0..usize::MAX, 0..usize::MAX); set_max_depth(32); + guard } fn boxed_frame_chain(ips: &[usize]) -> (Box<[Frame]>, usize) { @@ -529,7 +661,7 @@ mod tests { #[test] fn unwind_stack_collects_fake_frames() { - init_for_tests(); + let _serial = init_for_tests(); let (frames, start_fp) = boxed_frame_chain(&[0x1111, 0x2222, 0x3333]); let out = unwind_stack(start_fp); assert_eq!(out, frames.as_ref()); @@ -537,7 +669,7 @@ mod tests { #[test] fn unwind_core_callback_stop_early() { - init_for_tests(); + let _serial = init_for_tests(); let (_chain, start_fp) = boxed_frame_chain(&[0x1, 0x2, 0x3, 0x4, 0x5]); let mut count = 0; unwind_core(start_fp, |_| { @@ -549,7 +681,7 @@ mod tests { #[test] fn unwind_stack_stops_on_non_advancing_frame_pointer() { - init_for_tests(); + let _serial = init_for_tests(); let mut frames = [Frame { fp: 0, ip: 0x1111 }, Frame { fp: 0, ip: 0x2222 }]; let base = frames.as_mut_ptr(); frames[0].fp = unsafe { base.add(1) as usize }; @@ -566,11 +698,167 @@ mod tests { assert!(Frame::read(3).is_none()); } + // --- walk_fp (alloc-free, reader-injected) internal tests --- + + /// Reads a machine word straight out of the synthetic in-memory chain. + fn mem_reader(va: usize) -> Option { + Some(unsafe { *(va as *const usize) }) + } + + #[test] + fn walk_fp_leaf_plus_full_chain() { + let (_chain, start_fp) = boxed_frame_chain(&[0x1111, 0x2222, 0x3333]); + let mut out = [0u64; 8]; + let n = walk_fp( + 0xF00, + start_fp, + &(0..usize::MAX), + &(0..usize::MAX), + mem_reader, + &mut out, + ); + assert_eq!(n, 4); + assert_eq!(&out[..4], &[0xF00, 0x1111, 0x2222, 0x3333]); + } + + #[test] + fn walk_fp_skips_ip_outside_range_but_keeps_walking() { + let (_chain, start_fp) = boxed_frame_chain(&[0x1111, 0x2222]); + let mut out = [0u64; 8]; + // Only 0x2222 falls in the IP filter; 0x1111 is skipped, not recorded, + // yet the walk continues to the next frame. + let n = walk_fp( + 0xF00, + start_fp, + &(0x2000..0x3000), + &(0..usize::MAX), + mem_reader, + &mut out, + ); + assert_eq!(n, 2); + assert_eq!(&out[..2], &[0xF00, 0x2222]); + } + + #[test] + fn walk_fp_stops_on_non_advancing_fp() { + // frames[0].fp -> frames[1]; frames[1].fp -> frames[0] (backward). + let mut frames = [Frame { fp: 0, ip: 0x1111 }, Frame { fp: 0, ip: 0x2222 }]; + let base = frames.as_mut_ptr(); + frames[0].fp = unsafe { base.add(1) as usize }; + frames[1].fp = base as usize; // backward: read by the walk via `base` + let mut out = [0u64; 8]; + let n = walk_fp( + 0xF00, + base as usize, + &(0..usize::MAX), + &(0..usize::MAX), + mem_reader, + &mut out, + ); + // frames[1] points back to frames[0], so the walk must stop there. + assert_eq!(frames[1].fp, base as usize); + // Records the leaf and frames[0].ip, then the backward fp ends it. + assert_eq!(n, 2); + assert_eq!(&out[..2], &[0xF00, 0x1111]); + } + + #[test] + fn walk_fp_honors_out_capacity() { + let (_chain, start_fp) = boxed_frame_chain(&[0x1, 0x2, 0x3, 0x4, 0x5]); + let mut out = [0u64; 3]; // room for the leaf + 2 return addresses only + let n = walk_fp( + 0xF00, + start_fp, + &(0..usize::MAX), + &(0..usize::MAX), + mem_reader, + &mut out, + ); + assert_eq!(n, 3); + assert_eq!(&out[..3], &[0xF00, 0x1, 0x2]); + } + + #[test] + fn walk_fp_stops_when_reader_fails() { + let mut out = [0u64; 8]; + // An aligned, in-range fp whose read fails yields the leaf only. + let n = walk_fp( + 0xF00, + 0x1000, + &(0..usize::MAX), + &(0..usize::MAX), + |_| None, + &mut out, + ); + assert_eq!(n, 1); + assert_eq!(out[0], 0xF00); + } + + #[test] + fn walk_fp_rejects_out_of_range_or_misaligned_fp() { + let mut out = [0u64; 8]; + // fp outside fp_range -> leaf only. + let n = walk_fp( + 0xF00, + 0x9000, + &(0..usize::MAX), + &(0..0x1000), + mem_reader, + &mut out, + ); + assert_eq!(n, 1); + // misaligned fp -> leaf only (odd address is never a valid frame record). + let n = walk_fp( + 0xF00, + 0x1001, + &(0..usize::MAX), + &(0..usize::MAX), + mem_reader, + &mut out, + ); + assert_eq!(n, 1); + } + + #[test] + fn walk_fp_empty_out_returns_zero() { + let n = walk_fp( + 0xF00, + 0x1000, + &(0..usize::MAX), + &(0..usize::MAX), + mem_reader, + &mut [], + ); + assert_eq!(n, 0); + } + + #[test] + fn walk_fp_step_cap_terminates_all_skip_chain() { + // An ever-advancing chain where every return address (`[fp+8]`) is out of + // `ip_range`, so nothing is ever recorded and `n` never grows. The reader + // fabricates `[fp] = fp + 16` (a monotonic, always-mapped caller) and + // `[fp+8] = 0`. Without the total-step cap the loop would spin forever (a + // hang here would be the DoS the cap prevents); with it, the walk + // terminates and records only the leaf. + let reader = |va: usize| Some(if va.is_multiple_of(16) { va + 16 } else { 0 }); + let mut out = [0u64; 4]; + let n = walk_fp( + 0xF00, + 0x1000, + &(1..usize::MAX), + &(0..usize::MAX), + reader, + &mut out, + ); + assert_eq!(n, 1); + assert_eq!(out[0], 0xF00); + } + // --- capture_trap with Inner::Captured verification --- #[test] fn capture_trap_ra_not_substituted_with_wide_range() { - init_for_tests(); + let _serial = init_for_tests(); let (_chain, start_fp) = boxed_frame_chain(&[0xDEAD]); let bt = Backtrace::capture_trap(start_fp, 0x1000, 0xBEEF); let Inner::Captured(frames) = &bt.inner else { @@ -586,7 +874,7 @@ mod tests { /// Then unwind and verify every frame is collected. #[test] fn stress_fill_buffer_exactly() { - init_for_tests(); + let _serial = init_for_tests(); let ips: Vec = (0..CAPTURE_CAPACITY).map(|i| 0xA000 + i).collect(); let (chain, start_fp) = boxed_frame_chain(&ips); let out = unwind_stack(start_fp); @@ -598,7 +886,7 @@ mod tests { /// The trap frame is inserted at front, total = CAPTURE_CAPACITY, no eviction. #[test] fn stress_trap_near_capacity() { - init_for_tests(); + let _serial = init_for_tests(); let n = CAPTURE_CAPACITY - 1; let ips: Vec = (0..n).map(|i| 0xB000 + i).collect(); let (_chain, start_fp) = boxed_frame_chain(&ips); @@ -620,7 +908,7 @@ mod tests { /// The trap insert_front evicts the deepest frame. #[test] fn stress_trap_overflow_evicts_deepest() { - init_for_tests(); + let _serial = init_for_tests(); let ips: Vec = (0..CAPTURE_CAPACITY).map(|i| 0xD000 + i).collect(); let (_chain, start_fp) = boxed_frame_chain(&ips); @@ -641,7 +929,7 @@ mod tests { /// Build a chain deeper than max_depth and verify truncation. #[test] fn stress_deep_chain_truncation() { - init_for_tests(); + let _serial = init_for_tests(); set_max_depth(16); let ips: Vec = (0..64).map(|i| 0xF000 + i).collect(); let (chain, start_fp) = boxed_frame_chain(&ips); @@ -658,7 +946,7 @@ mod tests { /// Repeatedly create and drop Backtrace objects to verify no leaks or corruption. #[test] fn stress_repeated_create_drop() { - init_for_tests(); + let _serial = init_for_tests(); let (chain, start_fp) = boxed_frame_chain(&[0x100, 0x200, 0x300]); for _ in 0..500 { let bt = Backtrace::capture_trap(start_fp, 0x400, 0); @@ -675,7 +963,7 @@ mod tests { /// Interleave capture, Display formatting, and drop to verify no side effects. #[test] fn stress_interleaved_capture_format() { - init_for_tests(); + let _serial = init_for_tests(); let (chain, start_fp) = boxed_frame_chain(&[0x500, 0x600]); for i in 0..100 { @@ -701,7 +989,7 @@ mod tests { /// Repeatedly clone a Backtrace and verify equality. #[test] fn stress_repeated_clone() { - init_for_tests(); + let _serial = init_for_tests(); let (chain, start_fp) = boxed_frame_chain(&[0x800, 0x900, 0xA00]); let original = Backtrace::capture_trap(start_fp, 0xB00, 0); diff --git a/components/axcpu/Cargo.toml b/components/axcpu/Cargo.toml index c3eb0d80ee..e514c9e883 100644 --- a/components/axcpu/Cargo.toml +++ b/components/axcpu/Cargo.toml @@ -46,6 +46,7 @@ x86_64 = { version = "0.15", default-features = false, features = [ [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "11.0" tock-registers = "0.10" +ax-percpu = { workspace = true } [target.'cfg(any(target_arch = "riscv32", target_arch = "riscv64"))'.dependencies] riscv.workspace = true diff --git a/components/axcpu/src/aarch64/pmu.rs b/components/axcpu/src/aarch64/pmu.rs index 25d1a6c273..c7ade89be6 100644 --- a/components/axcpu/src/aarch64/pmu.rs +++ b/components/axcpu/src/aarch64/pmu.rs @@ -12,6 +12,8 @@ use core::arch::asm; +use super::TrapFrame; + /// Information probed from the PMU. pub struct PmuInfo { /// `PMCR_EL0.N`: number of programmable event counters. @@ -692,3 +694,86 @@ pub fn interrupted_is_user() -> bool { } (spsr & 0xf) == 0 } + +/// Per-CPU pointer to the [`TrapFrame`] of the context interrupted by the IRQ +/// currently being dispatched, or `0` when not inside an IRQ dispatch. +/// +/// Published at the two IRQ-entry sites (EL1 kernel interrupt, EL0 user +/// interrupt) immediately before `dispatch_irq` and cleared immediately after, +/// so the PMU overflow handler — which runs *inside* `dispatch_irq` — can read +/// the interrupted frame pointer (`x29`) for `PERF_SAMPLE_CALLCHAIN` unwinding. +/// The live `ELR_EL1`/`SPSR_EL1` (see [`interrupted_pc`]/[`interrupted_is_user`]) +/// still describe the interrupted PC and EL, but `x29` survives only in the +/// saved frame — the handler's own frames have long since clobbered the GPR. +#[ax_percpu::def_percpu] +static PMU_TRAP_FRAME: usize = 0; + +/// Publishes `tf` as the interrupted trap frame for the current CPU. +/// +/// Called at IRQ entry, immediately before `dispatch_irq`, and paired with +/// [`clear_trap_frame`] immediately after it returns. A single per-CPU store +/// (TPIDR-relative) that cannot fault and does not touch the register +/// save/restore path. +/// +/// # Safety +/// `tf` must point to a valid [`TrapFrame`] that stays alive until +/// [`clear_trap_frame`] runs on this CPU (i.e. for the duration of the IRQ +/// dispatch). Interrupts must remain masked across the publish → dispatch → +/// clear window so no nested IRQ observes this CPU's pointer for a frame that +/// has already returned. +#[inline] +pub unsafe fn set_trap_frame(tf: *const TrapFrame) { + PMU_TRAP_FRAME.write_current(tf as usize); +} + +/// Clears the published interrupted trap frame for the current CPU. +/// +/// Called immediately after `dispatch_irq` returns, so no later sampling +/// interrupt can observe a stale (already-returned) frame. +#[inline] +pub fn clear_trap_frame() { + PMU_TRAP_FRAME.write_current(0); +} + +/// The interrupted frame pointer (`x29`), from the trap frame published at IRQ +/// entry. +/// +/// Returns `None` when no frame is published — e.g. the overflow was taken on a +/// path that does not plumb the frame (synchronous exceptions), or before the +/// plumbing ran. Callers must fall back to a leaf-only callchain in that case +/// rather than skipping the sample. +pub fn interrupted_fp() -> Option { + let p = PMU_TRAP_FRAME.read_current(); + if p == 0 { + return None; + } + // SAFETY: `p` was published from a live `&TrapFrame` by `set_trap_frame` and + // is only non-zero for the duration of the IRQ dispatch on this CPU, during + // which we are running. + let tf = p as *const TrapFrame; + Some(unsafe { (*tf).x[29] as usize }) +} + +/// The interrupted stack pointer. +/// +/// For a user (EL0) interrupt the interrupted SP lives in `SP_EL0`, which the +/// kernel never overwrites (it runs on `SP_EL1`), so it still holds the user SP +/// and is read live. For a kernel (EL1) interrupt it is the pre-trap SP saved in +/// the published frame. Returns `None` if no frame is published on the kernel +/// path. Used only to bound the user-stack unwind window. +pub fn interrupted_sp() -> Option { + if interrupted_is_user() { + let sp: u64; + unsafe { + asm!("mrs {}, SP_EL0", out(reg) sp); + } + return Some(sp as usize); + } + let p = PMU_TRAP_FRAME.read_current(); + if p == 0 { + return None; + } + // SAFETY: see `interrupted_fp`. + let tf = p as *const TrapFrame; + Some(unsafe { (*tf).sp as usize }) +} diff --git a/components/axcpu/src/aarch64/trap.rs b/components/axcpu/src/aarch64/trap.rs index 5e6b94d046..a2cf8b5b27 100644 --- a/components/axcpu/src/aarch64/trap.rs +++ b/components/axcpu/src/aarch64/trap.rs @@ -128,7 +128,15 @@ fn aarch64_trap_handler(tf: &mut TrapFrame, kind: TrapKind, source: TrapSource) ); } TrapKind::Irq => { + // Publish the interrupted frame so a PMU overflow handler running + // inside `dispatch_irq` can unwind the interrupted call stack for + // `PERF_SAMPLE_CALLCHAIN`. Purely additive: set after SAVE_REGS, clear + // before returning, both under the IRQ-masked dispatch window, so no + // nested IRQ observes a stale frame and the save/restore path is + // untouched. + unsafe { super::pmu::set_trap_frame(tf as *const _) }; crate::trap::dispatch_irq(0); + super::pmu::clear_trap_frame(); } TrapKind::Synchronous => { #[cfg(not(feature = "arm-el2"))] diff --git a/components/axcpu/src/aarch64/uspace.rs b/components/axcpu/src/aarch64/uspace.rs index 0aa1897559..2d5a13034c 100644 --- a/components/axcpu/src/aarch64/uspace.rs +++ b/components/axcpu/src/aarch64/uspace.rs @@ -183,7 +183,15 @@ impl UserContext { let ret = match kind { TrapKind::Irq => { + // See the EL1 site in trap.rs: publish the interrupted user frame + // so a PMU overflow handler running inside `dispatch_irq` can + // unwind the user call stack for `PERF_SAMPLE_CALLCHAIN`. The user + // `x29` lives in `self.tf.x[29]`; `SP_EL0` still holds the user SP. + // IRQs stay masked from `enter_user`'s trap until `enable_irqs` + // below, so no nested IRQ observes a stale frame. + unsafe { super::pmu::set_trap_frame(&self.tf as *const _) }; crate::trap::dispatch_irq(0); + super::pmu::clear_trap_frame(); ReturnReason::Interrupt } TrapKind::Fiq | TrapKind::SError => ReturnReason::Unknown, diff --git a/os/StarryOS/kernel/src/perf/mod.rs b/os/StarryOS/kernel/src/perf/mod.rs index b3c4bfd808..0fe12c48ea 100644 --- a/os/StarryOS/kernel/src/perf/mod.rs +++ b/os/StarryOS/kernel/src/perf/mod.rs @@ -9,6 +9,11 @@ pub mod bpf; pub mod hw; pub mod kprobe; +/// IRQ-safe no-fault user/kernel memory reader for PMU-sampling FP unwinding. +/// ARM PMUv3 only; walks `TTBR0`/`TTBR1` against the direct map so a bad frame +/// pointer never faults. +#[cfg(target_arch = "aarch64")] +pub mod nofault; /// Per-CPU hardware-PMU state (allocator, cluster identity). ARM PMUv3 only; /// the per-core counter pools + cluster classification live here. #[cfg(target_arch = "aarch64")] @@ -33,6 +38,11 @@ pub mod task; #[cfg(target_arch = "aarch64")] pub mod tick; pub mod tracepoint; +/// Frame-pointer call-graph unwinding for PMU sampling (`PERF_SAMPLE_CALLCHAIN`). +/// ARM PMUv3 only; consumes the interrupted frame pointer plumbed through +/// `ax_cpu::pmu` and the alloc-free `axbacktrace::walk_fp` engine. +#[cfg(target_arch = "aarch64")] +pub mod unwind; pub mod uprobe; use alloc::{borrow::Cow, boxed::Box, sync::Arc, vec}; diff --git a/os/StarryOS/kernel/src/perf/nofault.rs b/os/StarryOS/kernel/src/perf/nofault.rs new file mode 100644 index 0000000000..e03195587b --- /dev/null +++ b/os/StarryOS/kernel/src/perf/nofault.rs @@ -0,0 +1,144 @@ +//! IRQ-safe, allocation-free, no-fault readers for the FP unwinder. +//! +//! The PMU overflow handler unwinds the interrupted stack from hard-IRQ context, +//! where a page fault is fatal (there is no kernel fault-fixup table). A frame +//! pointer taken from a corrupt or mid-prologue register may point anywhere, so +//! the stack must never be dereferenced directly — for *either* user or kernel +//! frames. Both [`read_user_word_nofault`] and [`read_kernel_word_nofault`] +//! instead walk the relevant translation table (`TTBR0_EL1` / `TTBR1_EL1`) by +//! hand against the always-mapped kernel direct map — every access is to a +//! page-table frame or the resolved target page reached through +//! [`phys_to_virt`](ax_runtime::hal::mem::phys_to_virt), never to the input VA — +//! so an unmapped or bogus address yields `None` rather than a fault. +//! +//! This mirrors `PageTable::query` (4 KiB granule, 4-level, 48-bit VA) but is +//! self-contained, takes no locks, and allocates nothing. An IRQ never switches +//! `TTBR0_EL1`/`TTBR1_EL1`, so the interrupted address space stays active while +//! the handler runs. + +use ax_memory_addr::PhysAddr; + +/// Descriptor is valid / present. +const PTE_VALID: u64 = 1 << 0; +/// Descriptor type bit: `1` = table pointer (levels 0-2) or page (level 3); +/// `0` = block (huge page) descriptor. +const PTE_TABLE_OR_PAGE: u64 = 1 << 1; +/// Output-address field of a descriptor: bits `[47:12]`. Masks off the low +/// attribute bits and the high `TTBR0` ASID bits. +const PTE_ADDR_MASK: u64 = 0x0000_ffff_ffff_f000; + +/// Whether physical address `pa` lies in normal, linear-mapped RAM. +/// +/// The kernel linear map also aliases device MMIO (`Device-nGnRE`), so +/// `phys_to_virt` alone does not distinguish RAM from a device register. Every +/// physical the walk is about to dereference (table frames *and* the resolved +/// leaf) is checked against the RAM ranges first, so a valid page-table entry +/// whose output address is device MMIO — reachable via a user `mmap` of an +/// accelerator register window (RGA/NPU/JPU) plus a wild frame pointer, or a +/// concurrently-torn descriptor on another CPU — can never trigger a live MMIO +/// read (device side effect) or a data abort (panic) from hard-IRQ context. +/// `phys_ram_ranges` is a tiny `'static` slice (lock-free, IRQ-safe). +#[inline] +fn phys_is_ram(pa: usize) -> bool { + ax_hal::mem::phys_ram_ranges() + .iter() + .any(|&(start, size)| pa >= start && pa - start < size) +} + +/// Reads the raw descriptor at `index` of the page-table frame at physical +/// address `table_pa`, through the direct map. `None` if `table_pa` is not RAM +/// (e.g. a stale/torn table pointer), so the deref cannot fault. +#[inline] +fn read_descriptor(table_pa: usize, index: usize) -> Option { + if !phys_is_ram(table_pa) { + return None; + } + let table_va = ax_runtime::hal::mem::phys_to_virt(PhysAddr::from(table_pa)).as_usize(); + // SAFETY: `table_pa` is confirmed RAM, so `table_va` is the direct-map virtual + // address of a mapped page-table frame; `index < 512` keeps the read inside the + // 4 KiB frame. Cannot fault. + Some(unsafe { *((table_va as *const u64).add(index)) }) +} + +/// Reads the 64-bit word at physical address `pa` through the direct map. `None` +/// if `pa` is not RAM (e.g. a device-MMIO leaf), so the read has no side effect +/// and cannot fault. +#[inline] +fn read_phys_word(pa: usize) -> Option { + if !phys_is_ram(pa) { + return None; + } + let va = ax_runtime::hal::mem::phys_to_virt(PhysAddr::from(pa)).as_usize(); + // SAFETY: `pa` is confirmed normal RAM, so `va` is the direct-map virtual + // address of a mapped page. The caller only passes 8-aligned addresses, so the + // 8-byte read stays within one page (4096 is a multiple of 8). The input + // virtual address itself is never dereferenced. + Some(unsafe { *(va as *const u64) }) +} + +/// Reads the 64-bit word at virtual address `va` by walking the translation +/// table rooted at physical address `root_pa`, entirely through the direct map. +/// +/// Returns the word at the resolved physical address if `va` maps to a present +/// page, or `None` for any not-present, reserved, or malformed translation. Never +/// dereferences `va` itself, so it cannot fault. `va` must be 8-byte aligned (an +/// 8-byte read at an 8-aligned address never straddles a page). +fn read_word_via(root_pa: usize, va: usize) -> Option { + // Root frame (mask off any ASID + attribute bits carried in the TTBR value). + let mut table_pa = (root_pa as u64 & PTE_ADDR_MASK) as usize; + + // 4-level walk: level 0 indexes VA[47:39], … level 3 indexes VA[20:12]. + for level in 0..4usize { + let shift = 39 - 9 * level; + let index = (va >> shift) & 0x1ff; + let pte = read_descriptor(table_pa, index)?; + if pte & PTE_VALID == 0 { + return None; + } + let table_or_page = pte & PTE_TABLE_OR_PAGE != 0; + if level < 3 { + if table_or_page { + // Table descriptor: descend. + table_pa = (pte & PTE_ADDR_MASK) as usize; + continue; + } + // Block (huge page) descriptor. A block is not permitted at level 0. + if level == 0 { + return None; + } + } else if !table_or_page { + // A level-3 descriptor with the type bit clear is reserved/invalid. + return None; + } + + // Leaf reached (4 KiB page at level 3, 2 MiB block at level 2, 1 GiB block + // at level 1). Compose the output physical address for `va`. + let offset_mask = (1usize << shift) - 1; + let base = (pte & PTE_ADDR_MASK) as usize & !offset_mask; + return read_phys_word(base | (va & offset_mask)); + } + + // Unreachable: the level-3 iteration always returns. + None +} + +/// Reads the 64-bit word at *user* virtual address `va` without ever faulting. +/// +/// Walks the active user translation table (`TTBR0_EL1`). See [`read_word_via`]. +/// Allocation-free, lock-free, and safe from hard-IRQ context; an IRQ never +/// switches `TTBR0_EL1`, so the interrupted user address space stays active. +pub fn read_user_word_nofault(va: usize) -> Option { + read_word_via(ax_cpu::asm::read_user_page_table().as_usize(), va) +} + +/// Reads the 64-bit word at *kernel* virtual address `va` without ever faulting. +/// +/// Walks the kernel translation table (`TTBR1_EL1`) the same way as +/// [`read_user_word_nofault`]. Used by the kernel FP unwinder so a corrupt or +/// stale in-kernel frame pointer that slips past the range/alignment guards and +/// points at an unmapped kernel VA yields `None` instead of an unrecoverable data +/// abort (kernel panic) in hard-IRQ context — kernel stacks are mapped, so a +/// legitimate frame still resolves and is read through the direct map. +pub fn read_kernel_word_nofault(va: usize) -> Option { + read_word_via(ax_cpu::asm::read_kernel_page_table().as_usize(), va) +} diff --git a/os/StarryOS/kernel/src/perf/sampling.rs b/os/StarryOS/kernel/src/perf/sampling.rs index 9789d13940..997029aa29 100644 --- a/os/StarryOS/kernel/src/perf/sampling.rs +++ b/os/StarryOS/kernel/src/perf/sampling.rs @@ -106,14 +106,17 @@ const PERF_RECORD_MISC_KERNEL: u16 = 1; /// `PERF_RECORD_MISC_USER`: the sample landed in user (EL0) context. const PERF_RECORD_MISC_USER: u16 = 2; -/// Upper bound on a single `PERF_RECORD_SAMPLE` we emit: 8-byte header plus at -/// most nine 8-byte scalar fields (IDENTIFIER, IP, TID(pid+tid), TIME, ADDR, ID, -/// STREAM_ID, CPU(cpu+res), PERIOD). [`build_sample`] writes into a stack buffer -/// of this size and returns the actual length. -const SAMPLE_RECORD_MAX_LEN: usize = 8 + 9 * 8; - -// `perf_event_sample_format` bits (see `man perf_event_open`). Only the scalar -// fields below are supported; every other bit (READ, CALLCHAIN, RAW, +/// Upper bound on a single `PERF_RECORD_SAMPLE` we emit: 8-byte header, at most +/// nine 8-byte scalar fields (IDENTIFIER, IP, TID(pid+tid), TIME, ADDR, ID, +/// STREAM_ID, CPU(cpu+res), PERIOD), then an optional callchain block — a `u64 +/// nr` count followed by up to two `PERF_CONTEXT_*` markers and `2 * +/// MAX_STACK_DEPTH` instruction pointers (a kernel region + a user region). +/// [`build_sample`] writes into a stack buffer of this size and returns the +/// actual length. +const SAMPLE_RECORD_MAX_LEN: usize = 8 + 9 * 8 + 8 + (2 + 2 * MAX_STACK_DEPTH) * 8; + +// `perf_event_sample_format` bits (see `man perf_event_open`). The scalar fields +// below plus `PERF_SAMPLE_CALLCHAIN` are supported; every other bit (READ, RAW, // BRANCH_STACK, REGS_USER/INTR, STACK_USER, WEIGHT, DATA_SRC, TRANSACTION, // PHYS_ADDR, …) is rejected at open time. /// `PERF_SAMPLE_IP`: instruction pointer. Always set by real `perf` for samples. @@ -134,11 +137,28 @@ const PERF_SAMPLE_PERIOD: u64 = 1 << 8; const PERF_SAMPLE_STREAM_ID: u64 = 1 << 9; /// `PERF_SAMPLE_IDENTIFIER`: leading event id (`u64`), emitted first. const PERF_SAMPLE_IDENTIFIER: u64 = 1 << 16; +/// `PERF_SAMPLE_CALLCHAIN`: per-sample call stack — a `u64 nr` count then `nr` +/// u64 instruction pointers, split into kernel/user regions by the +/// `PERF_CONTEXT_*` markers below. Set by `perf record -g` / `--call-graph fp`. +const PERF_SAMPLE_CALLCHAIN: u64 = 1 << 5; + +/// Callchain marker: the entries that follow are kernel (EL1) instruction +/// pointers (Linux `PERF_CONTEXT_KERNEL`). Counts as one callchain entry. +const PERF_CONTEXT_KERNEL: u64 = (-128i64) as u64; +/// Callchain marker: the entries that follow are user (EL0) instruction pointers +/// (Linux `PERF_CONTEXT_USER`). Counts as one callchain entry. +const PERF_CONTEXT_USER: u64 = (-512i64) as u64; + +/// Per-region cap on callchain depth (the kernel and user regions are bounded +/// separately). Sizes the fixed on-stack chain and record buffers, so it stays +/// allocation-free in the overflow handler. +const MAX_STACK_DEPTH: usize = 64; /// Every `sample_type` bit the sampling backend can emit a well-formed /// `PERF_RECORD_SAMPLE` for. A sampling event whose `sample_type` sets any bit /// outside this mask is rejected at open ([`super::hw`] reuses this constant); -/// real `perf record` sets `IP|TID|TIME|PERIOD`, all within the mask. +/// real `perf record` sets `IP|TID|TIME|PERIOD`, and `-g` adds `CALLCHAIN`, all +/// within the mask. pub const SUPPORTED_SAMPLE_TYPE: u64 = PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_TIME @@ -147,7 +167,8 @@ pub const SUPPORTED_SAMPLE_TYPE: u64 = PERF_SAMPLE_IP | PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD | PERF_SAMPLE_STREAM_ID - | PERF_SAMPLE_IDENTIFIER; + | PERF_SAMPLE_IDENTIFIER + | PERF_SAMPLE_CALLCHAIN; /// Everything the overflow handler needs for one counter, in a lock-free, /// alloc-free `Copy` POD. @@ -365,6 +386,15 @@ pub fn pmu_overflow_handler(_ctx: IrqContext) -> IrqReturn { let tid = ax_task::current().id().as_u64() as u32; let time = ax_runtime::hal::time::monotonic_time_nanos(); let cpu = ax_hal::percpu::this_cpu_id() as u32; + // Call stack for PERF_SAMPLE_CALLCHAIN (alloc-free, fixed on-stack buffer; + // empty unless the event requested it). Kernel frames are unwound from the + // interrupted x29; the user region is the leaf IP in M4a. + let mut chain = [0u64; 2 + 2 * MAX_STACK_DEPTH]; + let nchain = if sample_type & PERF_SAMPLE_CALLCHAIN != 0 { + build_callchain(ip, is_user, &mut chain) + } else { + 0 + }; let mut record = [0u8; SAMPLE_RECORD_MAX_LEN]; let data = SampleData { ip, @@ -376,6 +406,7 @@ pub fn pmu_overflow_handler(_ctx: IrqContext) -> IrqReturn { stream_id: 0, cpu, period: cur_period as u64, + callchain: &chain[..nchain], }; let len = build_sample(&mut record, sample_type, misc, &data); @@ -430,6 +461,48 @@ pub fn pmu_overflow_handler(_ctx: IrqContext) -> IrqReturn { IrqReturn::Handled } +/// Fills `chain` with the interrupted call stack for `PERF_SAMPLE_CALLCHAIN`, +/// returning the number of `u64` entries written. +/// +/// The layout mirrors Linux: a `PERF_CONTEXT_*` region marker followed by that +/// region's instruction pointers, leaf first — `[PERF_CONTEXT_KERNEL, ip, ra0, +/// …]` for a kernel sample, `[PERF_CONTEXT_USER, ip, ra0, …]` for a user sample. +/// Kernel frames are unwound from the interrupted `x29` via +/// [`super::unwind::kernel_callchain`], user frames via +/// [`super::unwind::user_callchain`] (through the IRQ-safe no-fault `TTBR0` +/// reader). If no frame pointer was published the region degrades to +/// `[marker, ip]` (never empty, so the sample is never dropped). Deep frames +/// appear only when the sampled code keeps frame pointers (the kernel when built +/// with `-Cforce-frame-pointers`, user binaries built `-fno-omit-frame-pointer`). +/// Allocation-free and safe from the overflow handler. +fn build_callchain(ip: u64, is_user: bool, chain: &mut [u64]) -> usize { + // `chain` is the handler's fixed `[u64; 2 + 2 * MAX_STACK_DEPTH]` buffer, so + // the leading fixed-index writes below are always in bounds. Each region is + // capped at `MAX_STACK_DEPTH` frames (plus its one-word marker). + chain[0] = if is_user { + PERF_CONTEXT_USER + } else { + PERF_CONTEXT_KERNEL + }; + let region_end = (1 + MAX_STACK_DEPTH).min(chain.len()); + let region = &mut chain[1..region_end]; + match ax_cpu::pmu::interrupted_fp() { + Some(fp) if is_user => { + // Bound the user walk to the interrupted user stack (SP_EL0); fall back + // to the frame pointer itself as the window anchor if unavailable. + let sp = ax_cpu::pmu::interrupted_sp().unwrap_or(fp); + 1 + super::unwind::user_callchain(ip as usize, fp, sp, region) + } + Some(fp) => 1 + super::unwind::kernel_callchain(ip as usize, fp, region), + None => { + // No frame pointer published (e.g. sampled on a path that does not + // plumb it): emit the leaf IP alone rather than dropping the region. + chain[1] = ip; + 2 + } + } +} + /// Lays out one `PERF_RECORD_SAMPLE` into `buf` per `sample_type`, returning its /// total length in bytes. /// @@ -447,13 +520,15 @@ pub fn pmu_overflow_handler(_ctx: IrqContext) -> IrqReturn { /// 8. `STREAM_ID` → `u64 stream_id` /// 9. `CPU` → `u32 cpu`, `u32 res = 0` /// 10. `PERIOD` → `u64 period` +/// 11. `CALLCHAIN` → `u64 nr`, then `nr` u64 entries (`PERF_CONTEXT_*` markers + +/// instruction pointers) from `d.callchain` /// /// `buf` must be at least [`SAMPLE_RECORD_MAX_LEN`] bytes. With /// `sample_type == PERF_SAMPLE_IP` exactly, the result is the original 16-byte /// IP-only record (8-byte header + `u64 ip`). -/// The per-sample scalar values [`build_sample`] may emit (those not implied by +/// The per-sample values [`build_sample`] may emit (those not implied by /// `sample_type` alone). Gathered by the overflow handler at interrupt time. -struct SampleData { +struct SampleData<'a> { ip: u64, pid: u32, tid: u32, @@ -463,11 +538,17 @@ struct SampleData { stream_id: u64, cpu: u32, period: u64, + /// The `PERF_SAMPLE_CALLCHAIN` entries (`PERF_CONTEXT_*` markers + IPs), or an + /// empty slice when the event did not request a callchain. Emitted verbatim as + /// the `nr` count followed by the entries. Borrows the handler's fixed on-stack + /// chain buffer. + callchain: &'a [u64], } -fn build_sample(buf: &mut [u8], sample_type: u64, misc: u16, d: &SampleData) -> usize { +fn build_sample(buf: &mut [u8], sample_type: u64, misc: u16, d: &SampleData<'_>) -> usize { // Cursor into `buf`. All offsets stay within `SAMPLE_RECORD_MAX_LEN` because - // at most the header + 9 u64-sized fields are written and the caller passes a + // at most the header + 9 u64 scalar fields + the callchain block (`nr` plus at + // most `2 + 2*MAX_STACK_DEPTH` entries) are written, and the caller passes a // buffer of that size. `put!` appends a native-endian scalar and advances the // cursor (a macro, not a closure, so it never holds a borrow of `off`). let mut off = 0usize; @@ -517,6 +598,14 @@ fn build_sample(buf: &mut [u8], sample_type: u64, misc: u16, d: &SampleData) -> if sample_type & PERF_SAMPLE_PERIOD != 0 { put!(d.period); } + if sample_type & PERF_SAMPLE_CALLCHAIN != 0 { + // `u64 nr` count (the `PERF_CONTEXT_*` markers count as entries) followed + // by the entries themselves, exactly as Linux lays out the block. + put!(d.callchain.len() as u64); + for &entry in d.callchain { + put!(entry); + } + } // Back-patch the header's `size` field now that the total length is known. buf[size_off..size_off + 2].copy_from_slice(&(off as u16).to_ne_bytes()); diff --git a/os/StarryOS/kernel/src/perf/unwind.rs b/os/StarryOS/kernel/src/perf/unwind.rs new file mode 100644 index 0000000000..ece82e7378 --- /dev/null +++ b/os/StarryOS/kernel/src/perf/unwind.rs @@ -0,0 +1,94 @@ +//! Kernel-side integration for PMU-sampling call-graph capture +//! (`PERF_SAMPLE_CALLCHAIN`). +//! +//! The allocation-free frame-pointer walk engine itself lives in `axbacktrace` +//! ([`axbacktrace::walk_fp`]); this module supplies the kernel address ranges and +//! the memory-read strategy used from the PMU overflow IRQ handler: +//! +//! * **Kernel frames** are read by a direct dereference — kernel stacks are +//! always mapped, and the IP/FP ranges reuse the ones `axbacktrace::init` +//! already records at boot (`[_stext, _etext)` / the kernel address space). +//! * **User frames** are read by the IRQ-safe no-fault page-table walk in +//! [`super::nofault`], seeded from the interrupted `SP_EL0`/`x29` (Task M4b). +//! +//! Everything here is called from hard-IRQ context, so it must be allocation-free +//! and take no sleeping locks. + +use core::ops::Range; + +/// The instruction (text) and frame-pointer ranges used to validate a *kernel* +/// frame-pointer walk. +/// +/// Reuses the ranges [`axbacktrace::init`] recorded at boot: `ip` = `[_stext, +/// _etext)`, `fp` = the kernel address space `[kernel_space_start, _end)`. Both +/// getters are lock-free (`spin::Once::get`), so this is safe from the overflow +/// handler. Returns `None` only before backtrace init (never on a running +/// system). +pub fn kernel_ranges() -> Option<(Range, Range)> { + Some((axbacktrace::ip_range()?, axbacktrace::fp_range()?)) +} + +/// Reads a machine word at a *kernel* virtual address for the FP walk. +/// +/// Routed through the IRQ-safe no-fault reader ([`super::nofault`]) rather than a +/// raw dereference: a corrupt or stale frame pointer that slips past the +/// range / alignment / monotonic guards and lands on an unmapped kernel VA +/// returns `None` (ending the walk) instead of taking an unrecoverable data abort +/// in hard-IRQ context (there is no kernel fault-fixup table). A legitimate +/// kernel-stack frame is mapped, so it resolves and is read through the direct +/// map. +#[inline] +fn read_kernel_word(va: usize) -> Option { + super::nofault::read_kernel_word_nofault(va).map(|w| w as usize) +} + +/// Walks the kernel frame-pointer chain from (`pc`, `fp`) into `out`, returning +/// the number of `u64` entries written (`out[0] == pc`, so always `>= 1` when +/// `out` is non-empty). +/// +/// Allocation-free and safe from the PMU overflow handler. If the backtrace +/// ranges are not yet initialized, emits just the leaf `pc`. +pub fn kernel_callchain(pc: usize, fp: usize, out: &mut [u64]) -> usize { + let Some((ip_range, fp_range)) = kernel_ranges() else { + if out.is_empty() { + return 0; + } + out[0] = pc as u64; + return 1; + }; + axbacktrace::walk_fp(pc, fp, &ip_range, &fp_range, read_kernel_word, out) +} + +/// Walks the *user* frame-pointer chain from (`pc`, `fp`) into `out`, returning +/// the number of `u64` entries written (`out[0] == pc`, so always `>= 1` when +/// `out` is non-empty). +/// +/// Reads user memory through the IRQ-safe no-fault reader ([`super::nofault`]), +/// which returns `None` for any unmapped or non-RAM address — so a wild user `fp` +/// can never fault the kernel or touch device MMIO. The frame-pointer range is +/// bounded to a generous window above the interrupted user SP (`sp`): the stack +/// grows down, so every frame record sits at an address `>= sp`, and a wild `fp` +/// outside the window is rejected before any page-table walk. The IP range stays +/// permissive (any user VA); the no-fault reader validates each read. Together +/// with `walk_fp`'s alignment / monotonic / 8 MiB-gap / depth / total-step guards +/// this keeps a crafted user chain from stalling the handler. Yields a deep chain +/// only when the sampled user binary keeps frame pointers. +pub fn user_callchain(pc: usize, fp: usize, sp: usize, out: &mut [u64]) -> usize { + /// Cap on how far above the interrupted SP a user frame record may sit — a + /// typical maximum user stack, so a corrupt `fp` can't range over the whole + /// address space. + const USER_STACK_WINDOW: usize = 8 * 1024 * 1024; + const USER_VA_END: usize = 1 << 48; + let lo = sp & !0xfff; + let hi = lo.saturating_add(USER_STACK_WINDOW).min(USER_VA_END); + let fp_range = lo..hi; + let ip_range = 1..USER_VA_END; + axbacktrace::walk_fp( + pc, + fp, + &ip_range, + &fp_range, + |va| super::nofault::read_user_word_nofault(va).map(|w| w as usize), + out, + ) +} diff --git a/test-suit/starryos/qemu/system/perf-hw-callchain-kernel/CMakeLists.txt b/test-suit/starryos/qemu/system/perf-hw-callchain-kernel/CMakeLists.txt new file mode 100644 index 0000000000..75c06a1c10 --- /dev/null +++ b/test-suit/starryos/qemu/system/perf-hw-callchain-kernel/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.20) +project(perf-hw-callchain-kernel C) +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) +# Plain C (no arch-specific asm), so it builds on every arch; main() skips as a +# pass on non-aarch64 since hardware-PMU sampling is ARM PMUv3 only. Frame +# pointers are kept so the kernel FP chain the test checks is well formed (they +# already are in the kernel build; this only affects the tiny test binary). +add_executable(perf-hw-callchain-kernel src/perf_hw_callchain_kernel.c) +target_compile_options(perf-hw-callchain-kernel PRIVATE + -Wall + -Wextra + -Werror + -fno-omit-frame-pointer +) +install(TARGETS perf-hw-callchain-kernel RUNTIME DESTINATION usr/bin/starry-test-suit) diff --git a/test-suit/starryos/qemu/system/perf-hw-callchain-kernel/src/perf_hw_callchain_kernel.c b/test-suit/starryos/qemu/system/perf-hw-callchain-kernel/src/perf_hw_callchain_kernel.c new file mode 100644 index 0000000000..21d745e0af --- /dev/null +++ b/test-suit/starryos/qemu/system/perf-hw-callchain-kernel/src/perf_hw_callchain_kernel.c @@ -0,0 +1,412 @@ +/* + * perf_hw_callchain_kernel.c -- perf_event_open(2) PERF_SAMPLE_CALLCHAIN test. + * + * Proves the kernel call-graph sampling path (M4a): a sampling event opened with + * PERF_SAMPLE_CALLCHAIN must, for a sample taken in kernel (EL1) context, emit a + * callchain block of `[PERF_CONTEXT_KERNEL, leaf_ip, ret0, ret1, ...]` unwound + * from the interrupted frame pointer -- not just the leaf IP. A host `perf + * report` renders such records as flamegraphs. + * + * 1. open a SAMPLING event: PERF_TYPE_RAW / config=0x11 (ARM CPU_CYCLES, which + * counts under QEMU TCG), fixed sample_period, sample_type = + * PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_TIME | PERF_SAMPLE_CALLCHAIN, + * 2. mmap (1 header page + 8 data pages), + * 3. ENABLE, then run a SYSCALL-HEAVY workload (read() a page from /dev/zero in + * a loop) so most PMU overflows fire while executing inside the kernel, with + * a deep, unwindable kernel stack; DISABLE, + * 4. walk the ring, parse each PERF_RECORD_SAMPLE body in sample_type order + * (ip, pid+tid, time, then the callchain: u64 nr followed by nr u64 entries), + * 5. classify each callchain by its PERF_CONTEXT_* markers and measure how many + * instruction pointers follow the kernel marker. + * + * SUCCESS == + * fd >= 0 AND mmap ok AND the ring is non-empty + * AND at least one PERF_RECORD_SAMPLE carries a well-formed callchain block + * (nr never overruns the record) + * AND at least one KERNEL-context callchain (PERF_CONTEXT_KERNEL) with a leaf IP + * is present -- i.e. kernel-context sampling emits the correct callchain + * record with the kernel region marker. + * This deliberately does NOT require a DEEP kernel chain: the default kernel is + * built without frame pointers (see the note near the assertions), so the kernel + * region is leaf-only. The perf-hw-callchain-user case proves the FP unwinder + * actually walks multiple frames on a frame-pointer-enabled binary. + * On success exactly one line `STARRY_PERF_CALLCHAIN_OK` is printed. + * + * All ABI structs are defined locally (no dependency) and + * are byte-identical to the perf-hw-sample case; only sample_type and the record + * body parsing differ. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PERF_TYPE_RAW +#define PERF_TYPE_RAW 4u +#endif +#ifndef ARM_PMU_EVT_CPU_CYCLES +#define ARM_PMU_EVT_CPU_CYCLES 0x11ull +#endif + +/* sample_type bits (see man perf_event_open). */ +#define PERF_SAMPLE_IP (1ull << 0) +#define PERF_SAMPLE_TID (1ull << 1) +#define PERF_SAMPLE_TIME (1ull << 2) +#define PERF_SAMPLE_CALLCHAIN (1ull << 5) + +/* + * Callchain context markers (Linux). Everything >= PERF_CONTEXT_MAX + * (== (uint64_t)-4095) is a marker, not an instruction pointer. + */ +#define PERF_CONTEXT_KERNEL ((uint64_t)-128) +#define PERF_CONTEXT_USER ((uint64_t)-512) +#define PERF_CONTEXT_MAX ((uint64_t)-4095) + +/* A single overflow every this-many events. */ +#define SAMPLE_PERIOD 100000ull + +#ifndef PERF_EVENT_IOC_ENABLE +#define PERF_EVENT_IOC_ENABLE 0x2400u +#endif +#ifndef PERF_EVENT_IOC_DISABLE +#define PERF_EVENT_IOC_DISABLE 0x2401u +#endif +#ifndef PERF_EVENT_IOC_RESET +#define PERF_EVENT_IOC_RESET 0x2403u +#endif +#ifndef PERF_RECORD_SAMPLE +#define PERF_RECORD_SAMPLE 9u +#endif + +struct perf_event_attr { + uint32_t type; + uint32_t size; + uint64_t config; + union { + uint64_t sample_period; + uint64_t sample_freq; + }; + uint64_t sample_type; + uint64_t read_format; + uint64_t flags; + union { + uint32_t wakeup_events; + uint32_t wakeup_watermark; + }; + uint32_t bp_type; + union { + uint64_t bp_addr; + uint64_t config1; + }; + union { + uint64_t bp_len; + uint64_t config2; + }; + uint64_t branch_sample_type; + uint64_t sample_regs_user; + uint32_t sample_stack_user; + int32_t clockid; + uint64_t sample_regs_intr; + uint32_t aux_watermark; + uint16_t sample_max_stack; + uint16_t __reserved_2; + uint32_t aux_sample_size; + uint32_t __reserved_3; +}; + +#define PERF_ATTR_FLAG_DISABLED (1ull << 0) + +struct perf_event_mmap_page { + uint32_t version; + uint32_t compat_version; + uint32_t lock; + uint32_t index; + int64_t offset; + uint64_t time_enabled; + uint64_t time_running; + union { + uint64_t capabilities; + struct { + uint64_t cap_bit0 : 1, cap_bit0_is_deprecated : 1, + cap_user_rdpmc : 1, cap_user_time : 1, cap_user_time_zero : 1, + cap_user_time_short : 1, cap_____res : 58; + }; + }; + uint16_t pmc_width; + uint16_t time_shift; + uint32_t time_mult; + uint64_t time_offset; + uint64_t time_zero; + uint32_t size; + uint32_t __reserved_1; + uint64_t time_cycles; + uint64_t time_mask; + uint8_t __reserved[928]; + uint64_t data_head; + uint64_t data_tail; + uint64_t data_offset; + uint64_t data_size; + uint64_t aux_head; + uint64_t aux_tail; + uint64_t aux_offset; + uint64_t aux_size; +}; + +_Static_assert(offsetof(struct perf_event_attr, sample_period) == 16, "off16"); +_Static_assert(offsetof(struct perf_event_attr, sample_type) == 24, "off24"); +_Static_assert(offsetof(struct perf_event_attr, read_format) == 32, "off32"); +_Static_assert(offsetof(struct perf_event_attr, flags) == 40, "off40"); +_Static_assert(offsetof(struct perf_event_mmap_page, data_head) == 1024, "dh"); +_Static_assert(offsetof(struct perf_event_mmap_page, data_tail) == 1032, "dt"); +_Static_assert(offsetof(struct perf_event_mmap_page, data_offset) == 1040, "do"); +_Static_assert(offsetof(struct perf_event_mmap_page, data_size) == 1048, "ds"); + +struct perf_event_header { + uint32_t type; + uint16_t misc; + uint16_t size; +}; + +#ifndef SYS_perf_event_open +#define SYS_perf_event_open 241 +#endif + +#define PERF_MMAP_PAGE_SIZE 4096u +#define PERF_MMAP_DATA_PAGES 8u +#define PERF_MMAP_TOTAL_BYTES \ + ((size_t)(1u + PERF_MMAP_DATA_PAGES) * PERF_MMAP_PAGE_SIZE) + +/* A hard ceiling on nr so a corrupt record cannot spin the parser. */ +#define CALLCHAIN_NR_MAX 512u + +static long perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu, + int group_fd, unsigned long flags) { + return syscall(SYS_perf_event_open, attr, pid, cpu, group_fd, flags); +} + +static int fail(const char *reason) { + printf("perf-callchain FAILED: %s\n", reason); + return 1; +} + +/* Copy `n` bytes out of the ring at byte offset `at`, wrapping modulo size. */ +static void ring_copy(const uint8_t *base, uint64_t size, uint64_t at, void *dst, + size_t n) { + for (size_t b = 0; b < n; b++) { + ((uint8_t *)dst)[b] = base[(at + b) % size]; + } +} + +int main(void) { +#if !defined(__aarch64__) + /* Hardware-PMU perf is aarch64-only (ARM PMUv3); skip-as-pass elsewhere. */ + printf("STARRY_PERF_CALLCHAIN_OK\n"); + return 0; +#endif + struct perf_event_attr attr; + for (size_t i = 0; i < sizeof(attr); i++) { + ((volatile unsigned char *)&attr)[i] = 0; + } + attr.type = PERF_TYPE_RAW; + attr.config = ARM_PMU_EVT_CPU_CYCLES; + attr.size = (uint32_t)sizeof(struct perf_event_attr); + attr.sample_period = SAMPLE_PERIOD; + attr.sample_type = + PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_TIME | PERF_SAMPLE_CALLCHAIN; + attr.read_format = 0; + attr.flags = PERF_ATTR_FLAG_DISABLED; + + long fd = perf_event_open(&attr, 0, -1, -1, 0ul); + if (fd < 0) { + char msg[96]; + snprintf(msg, sizeof(msg), "perf_event_open(callchain) errno=%d", errno); + return fail(msg); + } + int efd = (int)fd; + + void *base = mmap(NULL, PERF_MMAP_TOTAL_BYTES, PROT_READ | PROT_WRITE, + MAP_SHARED, efd, 0); + if (base == MAP_FAILED) { + int e = errno; + char msg[96]; + snprintf(msg, sizeof(msg), "mmap ring errno=%d", e); + close(efd); + return fail(msg); + } + struct perf_event_mmap_page *meta = (struct perf_event_mmap_page *)base; + + /* + * Syscall-heavy workload: read a page from /dev/zero in a tight loop so PMU + * overflows land inside the kernel with a deep, frame-pointer-walkable stack + * (sys_read -> VFS -> device). If /dev/zero is unavailable, fall back to a + * getpid() loop -- still kernel context, if a shallower stack. + */ + int zfd = open("/dev/zero", O_RDONLY); + + (void)ioctl(efd, PERF_EVENT_IOC_RESET, 0); + (void)ioctl(efd, PERF_EVENT_IOC_ENABLE, 0); + + if (zfd >= 0) { + static uint8_t buf[4096]; + for (uint64_t i = 0; i < 400000ull; i++) { + if (read(zfd, buf, sizeof(buf)) < 0) { + break; + } + } + } else { + volatile long p = 0; + for (uint64_t i = 0; i < 4000000ull; i++) { + p += syscall(SYS_getpid); + } + (void)p; + } + + (void)ioctl(efd, PERF_EVENT_IOC_DISABLE, 0); + if (zfd >= 0) { + close(zfd); + } + + uint64_t data_head = meta->data_head; + __sync_synchronize(); + uint64_t data_tail = meta->data_tail; + uint64_t data_offset = meta->data_offset; + uint64_t data_size = meta->data_size; + const uint8_t *data_base = (const uint8_t *)base + data_offset; + + uint64_t sample_count = 0; + uint64_t callchain_count = 0; /* samples carrying a callchain block */ + uint64_t kernel_chains = 0; /* callchains containing PERF_CONTEXT_KERNEL */ + uint64_t user_chains = 0; /* callchains containing PERF_CONTEXT_USER */ + uint64_t max_kernel_ips = 0; /* most IPs seen after PERF_CONTEXT_KERNEL */ + uint64_t max_user_ips = 0; /* most IPs seen after PERF_CONTEXT_USER */ + int saw_truncated = 0; + int bad_chain = 0; /* a callchain that ran past the record -> corrupt */ + + uint64_t off = data_tail; + while (off < data_head && data_size != 0) { + uint64_t rel = off % data_size; + struct perf_event_header hdr; + ring_copy(data_base, data_size, rel, &hdr, sizeof(hdr)); + + if (hdr.size == 0) { + saw_truncated = 1; + break; + } + if (off + hdr.size > data_head) { + saw_truncated = 1; + break; + } + + if (hdr.type == PERF_RECORD_SAMPLE) { + sample_count++; + /* + * Body layout for sample_type = IP|TID|TIME|CALLCHAIN, in the + * kernel's canonical field order: + * u64 ip; u32 pid; u32 tid; u64 time; u64 nr; u64 entries[nr]; + */ + uint64_t cur = (uint64_t)sizeof(hdr); /* offset within the record */ + cur += 8; /* ip */ + cur += 8; /* pid+tid */ + cur += 8; /* time */ + + if (cur + 8 <= hdr.size) { + uint64_t nr = 0; + ring_copy(data_base, data_size, (rel + cur) % data_size, &nr, 8); + cur += 8; + + /* nr entries must fit in the record and stay sane. */ + uint64_t avail = (hdr.size - cur) / 8; + if (nr > avail || nr > CALLCHAIN_NR_MAX) { + bad_chain = 1; + } else if (nr > 0) { + callchain_count++; + /* Count IPs following each context marker. */ + int in_kernel = 0, in_user = 0; + uint64_t k_ips = 0, u_ips = 0; + for (uint64_t e = 0; e < nr; e++) { + uint64_t entry = 0; + ring_copy(data_base, data_size, + (rel + cur + e * 8) % data_size, &entry, 8); + if (entry >= PERF_CONTEXT_MAX) { + /* A context marker: switch regions. */ + in_kernel = (entry == PERF_CONTEXT_KERNEL); + in_user = (entry == PERF_CONTEXT_USER); + if (in_kernel) { + kernel_chains++; + } + if (in_user) { + user_chains++; + } + } else if (in_kernel) { + k_ips++; + } else if (in_user) { + u_ips++; + } + } + if (k_ips > max_kernel_ips) { + max_kernel_ips = k_ips; + } + if (u_ips > max_user_ips) { + max_user_ips = u_ips; + } + } + } + } + + off += hdr.size; + } + + printf("STARRY_PERF_CALLCHAIN samples=%llu chains=%llu kchains=%llu " + "uchains=%llu max_kips=%llu max_uips=%llu truncated=%d bad=%d\n", + (unsigned long long)sample_count, (unsigned long long)callchain_count, + (unsigned long long)kernel_chains, (unsigned long long)user_chains, + (unsigned long long)max_kernel_ips, (unsigned long long)max_user_ips, + saw_truncated, bad_chain); + + int rc = 0; + if (fd < 0) { + rc = fail("fd is negative"); + } else if (base == MAP_FAILED) { + rc = fail("mmap failed"); + } else if (data_head == data_tail) { + rc = fail("no samples captured (data_head == data_tail)"); + } else if (sample_count == 0) { + rc = fail("no PERF_RECORD_SAMPLE records in ring"); + } else if (bad_chain) { + rc = fail("callchain nr overran the record (corrupt block)"); + } else if (callchain_count == 0) { + rc = fail("no sample carried a callchain block"); + } else if (kernel_chains == 0) { + rc = fail("no kernel-context callchain (no PERF_CONTEXT_KERNEL region)"); + } else if (max_kernel_ips < 1) { + rc = fail("kernel callchain carried no instruction pointer"); + } + /* + * NOTE: this validates the kernel callchain RECORD (block emitted, well + * formed, PERF_CONTEXT_KERNEL region with a leaf IP). It does NOT require a + * DEEP kernel chain, because the default kernel is built without frame + * pointers (like Linux without CONFIG_FRAME_POINTER), so x29 is not an + * unwindable chain and max_kernel_ips is 1. Deep kernel unwinding needs a + * kernel built with -Cforce-frame-pointers (BACKTRACE=y); the perf-hw- + * callchain-user case proves the FP walk engine itself unwinds multiple + * frames on a frame-pointer-enabled (user) binary. + */ + + (void)munmap(base, PERF_MMAP_TOTAL_BYTES); + close(efd); + + if (rc == 0) { + printf("STARRY_PERF_CALLCHAIN_OK\n"); + return 0; + } + return rc; +} diff --git a/test-suit/starryos/qemu/system/perf-hw-callchain-user/CMakeLists.txt b/test-suit/starryos/qemu/system/perf-hw-callchain-user/CMakeLists.txt new file mode 100644 index 0000000000..f75543f2e3 --- /dev/null +++ b/test-suit/starryos/qemu/system/perf-hw-callchain-user/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.20) +project(perf-hw-callchain-user C) +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) +# The test workload is a known nested call chain that the kernel unwinds from +# user space, so the binary MUST keep frame pointers and real (non-tail) calls. +# -O0 + -fno-omit-frame-pointer + -fno-optimize-sibling-calls guarantees each +# outer/mid/inner/busy frame has a walkable AAPCS64 frame record. Plain C (no +# arch asm), so it builds on every arch; main() skips as a pass off aarch64. +add_executable(perf-hw-callchain-user src/perf_hw_callchain_user.c) +target_compile_options(perf-hw-callchain-user PRIVATE + -Wall + -Wextra + -Werror + -O0 + -fno-omit-frame-pointer + -fno-optimize-sibling-calls +) +install(TARGETS perf-hw-callchain-user RUNTIME DESTINATION usr/bin/starry-test-suit) diff --git a/test-suit/starryos/qemu/system/perf-hw-callchain-user/src/perf_hw_callchain_user.c b/test-suit/starryos/qemu/system/perf-hw-callchain-user/src/perf_hw_callchain_user.c new file mode 100644 index 0000000000..3efb597251 --- /dev/null +++ b/test-suit/starryos/qemu/system/perf-hw-callchain-user/src/perf_hw_callchain_user.c @@ -0,0 +1,366 @@ +/* + * perf_hw_callchain_user.c -- PERF_SAMPLE_CALLCHAIN *user* frame-pointer test. + * + * Proves the kernel actually walks a USER stack (M4b): with a sampling event + * opened with PERF_SAMPLE_CALLCHAIN, samples taken while running a known nested + * call chain outer()->mid()->inner()->busy() (each __attribute__((noinline)), + * frame pointers kept) must emit `[PERF_CONTEXT_USER, busy_ip, inner_ret, + * mid_ret, outer_ret, ...]` -- i.e. the kernel unwound several user frames via + * the IRQ-safe no-fault TTBR0 reader, not just the leaf. This is the piece + * application-level flamegraphs need, and it works with the default (no-frame- + * pointer) kernel because the unwound stack is the frame-pointer-enabled test + * binary's own. + * + * Flow mirrors perf-hw-sample: open PERF_TYPE_RAW/config=0x11 sampling event with + * sample_type = IP|TID|TIME|CALLCHAIN, mmap the ring, ENABLE, run the nested + * busy loop, DISABLE, then parse each PERF_RECORD_SAMPLE body (ip, pid+tid, time, + * u64 nr, nr u64 entries) and measure the depth after PERF_CONTEXT_USER. + * + * SUCCESS == + * fd >= 0 AND mmap ok AND the ring is non-empty + * AND at least one well-formed callchain block (nr never overruns the record) + * AND at least one USER-context callchain has >= 4 instruction pointers after + * PERF_CONTEXT_USER (leaf busy() plus >=3 unwound return addresses through + * inner/mid/outer -- proving multi-frame user FP unwinding). + * On success exactly one line `STARRY_PERF_CALLCHAIN_USER_OK` is printed. + * + * All ABI structs are defined locally (no dependency). + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PERF_TYPE_RAW +#define PERF_TYPE_RAW 4u +#endif +#ifndef ARM_PMU_EVT_CPU_CYCLES +#define ARM_PMU_EVT_CPU_CYCLES 0x11ull +#endif + +#define PERF_SAMPLE_IP (1ull << 0) +#define PERF_SAMPLE_TID (1ull << 1) +#define PERF_SAMPLE_TIME (1ull << 2) +#define PERF_SAMPLE_CALLCHAIN (1ull << 5) + +#define PERF_CONTEXT_KERNEL ((uint64_t)-128) +#define PERF_CONTEXT_USER ((uint64_t)-512) +#define PERF_CONTEXT_MAX ((uint64_t)-4095) + +#define SAMPLE_PERIOD 100000ull + +#ifndef PERF_EVENT_IOC_ENABLE +#define PERF_EVENT_IOC_ENABLE 0x2400u +#endif +#ifndef PERF_EVENT_IOC_DISABLE +#define PERF_EVENT_IOC_DISABLE 0x2401u +#endif +#ifndef PERF_EVENT_IOC_RESET +#define PERF_EVENT_IOC_RESET 0x2403u +#endif +#ifndef PERF_RECORD_SAMPLE +#define PERF_RECORD_SAMPLE 9u +#endif + +struct perf_event_attr { + uint32_t type; + uint32_t size; + uint64_t config; + union { + uint64_t sample_period; + uint64_t sample_freq; + }; + uint64_t sample_type; + uint64_t read_format; + uint64_t flags; + union { + uint32_t wakeup_events; + uint32_t wakeup_watermark; + }; + uint32_t bp_type; + union { + uint64_t bp_addr; + uint64_t config1; + }; + union { + uint64_t bp_len; + uint64_t config2; + }; + uint64_t branch_sample_type; + uint64_t sample_regs_user; + uint32_t sample_stack_user; + int32_t clockid; + uint64_t sample_regs_intr; + uint32_t aux_watermark; + uint16_t sample_max_stack; + uint16_t __reserved_2; + uint32_t aux_sample_size; + uint32_t __reserved_3; +}; + +#define PERF_ATTR_FLAG_DISABLED (1ull << 0) + +struct perf_event_mmap_page { + uint32_t version; + uint32_t compat_version; + uint32_t lock; + uint32_t index; + int64_t offset; + uint64_t time_enabled; + uint64_t time_running; + union { + uint64_t capabilities; + struct { + uint64_t cap_bit0 : 1, cap_bit0_is_deprecated : 1, + cap_user_rdpmc : 1, cap_user_time : 1, cap_user_time_zero : 1, + cap_user_time_short : 1, cap_____res : 58; + }; + }; + uint16_t pmc_width; + uint16_t time_shift; + uint32_t time_mult; + uint64_t time_offset; + uint64_t time_zero; + uint32_t size; + uint32_t __reserved_1; + uint64_t time_cycles; + uint64_t time_mask; + uint8_t __reserved[928]; + uint64_t data_head; + uint64_t data_tail; + uint64_t data_offset; + uint64_t data_size; + uint64_t aux_head; + uint64_t aux_tail; + uint64_t aux_offset; + uint64_t aux_size; +}; + +_Static_assert(offsetof(struct perf_event_attr, sample_period) == 16, "off16"); +_Static_assert(offsetof(struct perf_event_attr, sample_type) == 24, "off24"); +_Static_assert(offsetof(struct perf_event_attr, read_format) == 32, "off32"); +_Static_assert(offsetof(struct perf_event_attr, flags) == 40, "off40"); +_Static_assert(offsetof(struct perf_event_mmap_page, data_head) == 1024, "dh"); +_Static_assert(offsetof(struct perf_event_mmap_page, data_tail) == 1032, "dt"); +_Static_assert(offsetof(struct perf_event_mmap_page, data_offset) == 1040, "do"); +_Static_assert(offsetof(struct perf_event_mmap_page, data_size) == 1048, "ds"); + +struct perf_event_header { + uint32_t type; + uint16_t misc; + uint16_t size; +}; + +#ifndef SYS_perf_event_open +#define SYS_perf_event_open 241 +#endif + +#define PERF_MMAP_PAGE_SIZE 4096u +#define PERF_MMAP_DATA_PAGES 8u +#define PERF_MMAP_TOTAL_BYTES \ + ((size_t)(1u + PERF_MMAP_DATA_PAGES) * PERF_MMAP_PAGE_SIZE) + +#define CALLCHAIN_NR_MAX 512u + +static long perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu, + int group_fd, unsigned long flags) { + return syscall(SYS_perf_event_open, attr, pid, cpu, group_fd, flags); +} + +static int fail(const char *reason) { + printf("perf-callchain-user FAILED: %s\n", reason); + return 1; +} + +static void ring_copy(const uint8_t *base, uint64_t size, uint64_t at, void *dst, + size_t n) { + for (size_t b = 0; b < n; b++) { + ((uint8_t *)dst)[b] = base[(at + b) % size]; + } +} + +/* The known nested call chain sampled below. volatile sink prevents the loop + * being optimised away even though the whole file is built at -O0. */ +static volatile uint64_t g_sink; +static int g_zfd = -1; + +__attribute__((noinline)) static void busy(void) { + /* + * A syscall-heavy loop (read a page from /dev/zero), NOT a pure-arithmetic + * loop: QEMU-TCG's cycle counter barely advances on pure ALU work, so a + * plain arithmetic loop overflows the sampling counter almost never (~1 + * sample). read() does real work the counter tracks, so the counter + * overflows steadily and most samples land here in userspace (giving the + * outer->mid->inner->busy user chain). Falls back to arithmetic only if + * /dev/zero is unavailable. + */ + static uint8_t buf[4096]; + for (uint64_t i = 0; i < 400000ull; i++) { + if (g_zfd >= 0) { + if (read(g_zfd, buf, sizeof(buf)) < 0) { + break; + } + } else { + g_sink += i * 3ull + 1ull; + } + } +} +__attribute__((noinline)) static void inner(void) { busy(); } +__attribute__((noinline)) static void mid(void) { inner(); } +__attribute__((noinline)) static void outer(void) { mid(); } + +int main(void) { +#if !defined(__aarch64__) + /* Hardware-PMU perf is aarch64-only (ARM PMUv3); skip-as-pass elsewhere. */ + printf("STARRY_PERF_CALLCHAIN_USER_OK\n"); + return 0; +#endif + struct perf_event_attr attr; + for (size_t i = 0; i < sizeof(attr); i++) { + ((volatile unsigned char *)&attr)[i] = 0; + } + attr.type = PERF_TYPE_RAW; + attr.config = ARM_PMU_EVT_CPU_CYCLES; + attr.size = (uint32_t)sizeof(struct perf_event_attr); + attr.sample_period = SAMPLE_PERIOD; + attr.sample_type = + PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_TIME | PERF_SAMPLE_CALLCHAIN; + attr.read_format = 0; + attr.flags = PERF_ATTR_FLAG_DISABLED; + + long fd = perf_event_open(&attr, 0, -1, -1, 0ul); + if (fd < 0) { + char msg[96]; + snprintf(msg, sizeof(msg), "perf_event_open(callchain) errno=%d", errno); + return fail(msg); + } + int efd = (int)fd; + + void *base = mmap(NULL, PERF_MMAP_TOTAL_BYTES, PROT_READ | PROT_WRITE, + MAP_SHARED, efd, 0); + if (base == MAP_FAILED) { + int e = errno; + char msg[96]; + snprintf(msg, sizeof(msg), "mmap ring errno=%d", e); + close(efd); + return fail(msg); + } + struct perf_event_mmap_page *meta = (struct perf_event_mmap_page *)base; + + g_zfd = open("/dev/zero", O_RDONLY); + + (void)ioctl(efd, PERF_EVENT_IOC_RESET, 0); + (void)ioctl(efd, PERF_EVENT_IOC_ENABLE, 0); + outer(); /* outer -> mid -> inner -> busy: most samples land in busy() */ + (void)ioctl(efd, PERF_EVENT_IOC_DISABLE, 0); + if (g_zfd >= 0) { + close(g_zfd); + } + + uint64_t data_head = meta->data_head; + __sync_synchronize(); + uint64_t data_tail = meta->data_tail; + uint64_t data_offset = meta->data_offset; + uint64_t data_size = meta->data_size; + const uint8_t *data_base = (const uint8_t *)base + data_offset; + + uint64_t sample_count = 0; + uint64_t callchain_count = 0; + uint64_t user_chains = 0; + uint64_t max_user_ips = 0; + int saw_truncated = 0; + int bad_chain = 0; + + uint64_t off = data_tail; + while (off < data_head && data_size != 0) { + uint64_t rel = off % data_size; + struct perf_event_header hdr; + ring_copy(data_base, data_size, rel, &hdr, sizeof(hdr)); + if (hdr.size == 0) { + saw_truncated = 1; + break; + } + if (off + hdr.size > data_head) { + saw_truncated = 1; + break; + } + if (hdr.type == PERF_RECORD_SAMPLE) { + sample_count++; + /* body: u64 ip; u32 pid; u32 tid; u64 time; u64 nr; u64 entries[nr] */ + uint64_t cur = (uint64_t)sizeof(hdr) + 8 + 8 + 8; + if (cur + 8 <= hdr.size) { + uint64_t nr = 0; + ring_copy(data_base, data_size, (rel + cur) % data_size, &nr, 8); + cur += 8; + uint64_t avail = (hdr.size - cur) / 8; + if (nr > avail || nr > CALLCHAIN_NR_MAX) { + bad_chain = 1; + } else if (nr > 0) { + callchain_count++; + int in_user = 0; + uint64_t u_ips = 0; + for (uint64_t e = 0; e < nr; e++) { + uint64_t entry = 0; + ring_copy(data_base, data_size, + (rel + cur + e * 8) % data_size, &entry, 8); + if (entry >= PERF_CONTEXT_MAX) { + in_user = (entry == PERF_CONTEXT_USER); + if (in_user) { + user_chains++; + } + } else if (in_user) { + u_ips++; + } + } + if (u_ips > max_user_ips) { + max_user_ips = u_ips; + } + } + } + } + off += hdr.size; + } + + printf("STARRY_PERF_CALLCHAIN_USER samples=%llu chains=%llu uchains=%llu " + "max_uips=%llu truncated=%d bad=%d sink=%llu\n", + (unsigned long long)sample_count, (unsigned long long)callchain_count, + (unsigned long long)user_chains, (unsigned long long)max_user_ips, + saw_truncated, bad_chain, (unsigned long long)g_sink); + + int rc = 0; + if (fd < 0) { + rc = fail("fd is negative"); + } else if (base == MAP_FAILED) { + rc = fail("mmap failed"); + } else if (data_head == data_tail) { + rc = fail("no samples captured (data_head == data_tail)"); + } else if (sample_count == 0) { + rc = fail("no PERF_RECORD_SAMPLE records in ring"); + } else if (bad_chain) { + rc = fail("callchain nr overran the record (corrupt block)"); + } else if (callchain_count == 0) { + rc = fail("no sample carried a callchain block"); + } else if (max_user_ips < 4) { + /* Leaf busy() + >=3 unwound return addresses (inner/mid/outer) proves the + * kernel walked several user frames via the no-fault TTBR0 reader. */ + rc = fail("no user callchain with >= 4 IPs (user FP unwind not observed)"); + } + + (void)munmap(base, PERF_MMAP_TOTAL_BYTES); + close(efd); + + if (rc == 0) { + printf("STARRY_PERF_CALLCHAIN_USER_OK\n"); + return 0; + } + return rc; +}