feat(starry): perf call-graph capture (PERF_SAMPLE_CALLCHAIN)

components/axcpu/Cargo.toml

@@ -46,6 +46,7 @@ x86_64 = { version = "0.15", default-features = false, features = [
 [target.'cfg(target_arch = "aarch64")'.dependencies]
 aarch64-cpu = "11.0"
 tock-registers = "0.10"
+ax-percpu = { workspace = true }
 
 [target.'cfg(any(target_arch = "riscv32", target_arch = "riscv64"))'.dependencies]
 riscv.workspace = true

components/axcpu/src/aarch64/pmu.rs

@@ -12,6 +12,8 @@
 
 use core::arch::asm;
 
+use super::TrapFrame;
+
 /// Information probed from the PMU.
 pub struct PmuInfo {
     /// `PMCR_EL0.N`: number of programmable event counters.
@@ -692,3 +694,86 @@ pub fn interrupted_is_user() -> bool {
     }
     (spsr & 0xf) == 0
 }
+
+/// Per-CPU pointer to the [`TrapFrame`] of the context interrupted by the IRQ
+/// currently being dispatched, or `0` when not inside an IRQ dispatch.
+///
+/// Published at the two IRQ-entry sites (EL1 kernel interrupt, EL0 user
+/// interrupt) immediately before `dispatch_irq` and cleared immediately after,
+/// so the PMU overflow handler — which runs *inside* `dispatch_irq` — can read
+/// the interrupted frame pointer (`x29`) for `PERF_SAMPLE_CALLCHAIN` unwinding.
+/// The live `ELR_EL1`/`SPSR_EL1` (see [`interrupted_pc`]/[`interrupted_is_user`])
+/// still describe the interrupted PC and EL, but `x29` survives only in the
+/// saved frame — the handler's own frames have long since clobbered the GPR.
+#[ax_percpu::def_percpu]
+static PMU_TRAP_FRAME: usize = 0;
+
+/// Publishes `tf` as the interrupted trap frame for the current CPU.
+///
+/// Called at IRQ entry, immediately before `dispatch_irq`, and paired with
+/// [`clear_trap_frame`] immediately after it returns. A single per-CPU store
+/// (TPIDR-relative) that cannot fault and does not touch the register
+/// save/restore path.
+///
+/// # Safety
+/// `tf` must point to a valid [`TrapFrame`] that stays alive until
+/// [`clear_trap_frame`] runs on this CPU (i.e. for the duration of the IRQ
+/// dispatch). Interrupts must remain masked across the publish → dispatch →
+/// clear window so no nested IRQ observes this CPU's pointer for a frame that
+/// has already returned.
+#[inline]
+pub unsafe fn set_trap_frame(tf: *const TrapFrame) {
+    PMU_TRAP_FRAME.write_current(tf as usize);
+}
+
+/// Clears the published interrupted trap frame for the current CPU.
+///
+/// Called immediately after `dispatch_irq` returns, so no later sampling
+/// interrupt can observe a stale (already-returned) frame.
+#[inline]
+pub fn clear_trap_frame() {
+    PMU_TRAP_FRAME.write_current(0);
+}
+
+/// The interrupted frame pointer (`x29`), from the trap frame published at IRQ
+/// entry.
+///
+/// Returns `None` when no frame is published — e.g. the overflow was taken on a
+/// path that does not plumb the frame (synchronous exceptions), or before the
+/// plumbing ran. Callers must fall back to a leaf-only callchain in that case
+/// rather than skipping the sample.
+pub fn interrupted_fp() -> Option<usize> {
+    let p = PMU_TRAP_FRAME.read_current();
+    if p == 0 {
+        return None;
+    }
+    // SAFETY: `p` was published from a live `&TrapFrame` by `set_trap_frame` and
+    // is only non-zero for the duration of the IRQ dispatch on this CPU, during
+    // which we are running.
+    let tf = p as *const TrapFrame;
+    Some(unsafe { (*tf).x[29] as usize })
+}
+
+/// The interrupted stack pointer.
+///
+/// For a user (EL0) interrupt the interrupted SP lives in `SP_EL0`, which the
+/// kernel never overwrites (it runs on `SP_EL1`), so it still holds the user SP
+/// and is read live. For a kernel (EL1) interrupt it is the pre-trap SP saved in
+/// the published frame. Returns `None` if no frame is published on the kernel
+/// path. Used only to bound the user-stack unwind window.
+pub fn interrupted_sp() -> Option<usize> {
+    if interrupted_is_user() {
+        let sp: u64;
+        unsafe {
+            asm!("mrs {}, SP_EL0", out(reg) sp);
+        }
+        return Some(sp as usize);
+    }
+    let p = PMU_TRAP_FRAME.read_current();
+    if p == 0 {
+        return None;
+    }
+    // SAFETY: see `interrupted_fp`.
+    let tf = p as *const TrapFrame;
+    Some(unsafe { (*tf).sp as usize })
+}

components/axcpu/src/aarch64/trap.rs

@@ -128,7 +128,15 @@ fn aarch64_trap_handler(tf: &mut TrapFrame, kind: TrapKind, source: TrapSource)
             );
         }
         TrapKind::Irq => {
+            // Publish the interrupted frame so a PMU overflow handler running
+            // inside `dispatch_irq` can unwind the interrupted call stack for
+            // `PERF_SAMPLE_CALLCHAIN`. Purely additive: set after SAVE_REGS, clear
+            // before returning, both under the IRQ-masked dispatch window, so no
+            // nested IRQ observes a stale frame and the save/restore path is
+            // untouched.
+            unsafe { super::pmu::set_trap_frame(tf as *const _) };
             crate::trap::dispatch_irq(0);
+            super::pmu::clear_trap_frame();
         }
         TrapKind::Synchronous => {
             #[cfg(not(feature = "arm-el2"))]

components/axcpu/src/aarch64/uspace.rs

@@ -183,7 +183,15 @@ impl UserContext {
 
         let ret = match kind {
             TrapKind::Irq => {
+                // See the EL1 site in trap.rs: publish the interrupted user frame
+                // so a PMU overflow handler running inside `dispatch_irq` can
+                // unwind the user call stack for `PERF_SAMPLE_CALLCHAIN`. The user
+                // `x29` lives in `self.tf.x[29]`; `SP_EL0` still holds the user SP.
+                // IRQs stay masked from `enter_user`'s trap until `enable_irqs`
+                // below, so no nested IRQ observes a stale frame.
+                unsafe { super::pmu::set_trap_frame(&self.tf as *const _) };
                 crate::trap::dispatch_irq(0);
+                super::pmu::clear_trap_frame();
                 ReturnReason::Interrupt
             }
             TrapKind::Fiq | TrapKind::SError => ReturnReason::Unknown,

os/StarryOS/kernel/src/perf/mod.rs

@@ -9,6 +9,11 @@
 pub mod bpf;
 pub mod hw;
 pub mod kprobe;
+/// IRQ-safe no-fault user/kernel memory reader for PMU-sampling FP unwinding.
+/// ARM PMUv3 only; walks `TTBR0`/`TTBR1` against the direct map so a bad frame
+/// pointer never faults.
+#[cfg(target_arch = "aarch64")]
+pub mod nofault;
 /// Per-CPU hardware-PMU state (allocator, cluster identity). ARM PMUv3 only;
 /// the per-core counter pools + cluster classification live here.
 #[cfg(target_arch = "aarch64")]
@@ -33,6 +38,11 @@ pub mod task;
 #[cfg(target_arch = "aarch64")]
 pub mod tick;
 pub mod tracepoint;
+/// Frame-pointer call-graph unwinding for PMU sampling (`PERF_SAMPLE_CALLCHAIN`).
+/// ARM PMUv3 only; consumes the interrupted frame pointer plumbed through
+/// `ax_cpu::pmu` and the alloc-free `axbacktrace::walk_fp` engine.
+#[cfg(target_arch = "aarch64")]
+pub mod unwind;
 pub mod uprobe;
 
 use alloc::{borrow::Cow, boxed::Box, sync::Arc, vec};

os/StarryOS/kernel/src/perf/nofault.rs

@@ -0,0 +1,144 @@
+//! IRQ-safe, allocation-free, no-fault readers for the FP unwinder.
+//!
+//! The PMU overflow handler unwinds the interrupted stack from hard-IRQ context,
+//! where a page fault is fatal (there is no kernel fault-fixup table). A frame
+//! pointer taken from a corrupt or mid-prologue register may point anywhere, so
+//! the stack must never be dereferenced directly — for *either* user or kernel
+//! frames. Both [`read_user_word_nofault`] and [`read_kernel_word_nofault`]
+//! instead walk the relevant translation table (`TTBR0_EL1` / `TTBR1_EL1`) by
+//! hand against the always-mapped kernel direct map — every access is to a
+//! page-table frame or the resolved target page reached through
+//! [`phys_to_virt`](ax_runtime::hal::mem::phys_to_virt), never to the input VA —
+//! so an unmapped or bogus address yields `None` rather than a fault.
+//!
+//! This mirrors `PageTable::query` (4 KiB granule, 4-level, 48-bit VA) but is
+//! self-contained, takes no locks, and allocates nothing. An IRQ never switches
+//! `TTBR0_EL1`/`TTBR1_EL1`, so the interrupted address space stays active while
+//! the handler runs.
+
+use ax_memory_addr::PhysAddr;
+
+/// Descriptor is valid / present.
+const PTE_VALID: u64 = 1 << 0;
+/// Descriptor type bit: `1` = table pointer (levels 0-2) or page (level 3);
+/// `0` = block (huge page) descriptor.
+const PTE_TABLE_OR_PAGE: u64 = 1 << 1;
+/// Output-address field of a descriptor: bits `[47:12]`. Masks off the low
+/// attribute bits and the high `TTBR0` ASID bits.
+const PTE_ADDR_MASK: u64 = 0x0000_ffff_ffff_f000;
+
+/// Whether physical address `pa` lies in normal, linear-mapped RAM.
+///
+/// The kernel linear map also aliases device MMIO (`Device-nGnRE`), so
+/// `phys_to_virt` alone does not distinguish RAM from a device register. Every
+/// physical the walk is about to dereference (table frames *and* the resolved
+/// leaf) is checked against the RAM ranges first, so a valid page-table entry
+/// whose output address is device MMIO — reachable via a user `mmap` of an
+/// accelerator register window (RGA/NPU/JPU) plus a wild frame pointer, or a
+/// concurrently-torn descriptor on another CPU — can never trigger a live MMIO
+/// read (device side effect) or a data abort (panic) from hard-IRQ context.
+/// `phys_ram_ranges` is a tiny `'static` slice (lock-free, IRQ-safe).
+#[inline]
+fn phys_is_ram(pa: usize) -> bool {
+    ax_hal::mem::phys_ram_ranges()
+        .iter()
+        .any(|&(start, size)| pa >= start && pa - start < size)
+}
+
+/// Reads the raw descriptor at `index` of the page-table frame at physical
+/// address `table_pa`, through the direct map. `None` if `table_pa` is not RAM
+/// (e.g. a stale/torn table pointer), so the deref cannot fault.
+#[inline]
+fn read_descriptor(table_pa: usize, index: usize) -> Option<u64> {
+    if !phys_is_ram(table_pa) {
+        return None;
+    }
+    let table_va = ax_runtime::hal::mem::phys_to_virt(PhysAddr::from(table_pa)).as_usize();
+    // SAFETY: `table_pa` is confirmed RAM, so `table_va` is the direct-map virtual
+    // address of a mapped page-table frame; `index < 512` keeps the read inside the
+    // 4 KiB frame. Cannot fault.
+    Some(unsafe { *((table_va as *const u64).add(index)) })
+}
+
+/// Reads the 64-bit word at physical address `pa` through the direct map. `None`
+/// if `pa` is not RAM (e.g. a device-MMIO leaf), so the read has no side effect
+/// and cannot fault.
+#[inline]
+fn read_phys_word(pa: usize) -> Option<u64> {
+    if !phys_is_ram(pa) {
+        return None;
+    }
+    let va = ax_runtime::hal::mem::phys_to_virt(PhysAddr::from(pa)).as_usize();
+    // SAFETY: `pa` is confirmed normal RAM, so `va` is the direct-map virtual
+    // address of a mapped page. The caller only passes 8-aligned addresses, so the
+    // 8-byte read stays within one page (4096 is a multiple of 8). The input
+    // virtual address itself is never dereferenced.
+    Some(unsafe { *(va as *const u64) })
+}
+
+/// Reads the 64-bit word at virtual address `va` by walking the translation
+/// table rooted at physical address `root_pa`, entirely through the direct map.
+///
+/// Returns the word at the resolved physical address if `va` maps to a present
+/// page, or `None` for any not-present, reserved, or malformed translation. Never
+/// dereferences `va` itself, so it cannot fault. `va` must be 8-byte aligned (an
+/// 8-byte read at an 8-aligned address never straddles a page).
+fn read_word_via(root_pa: usize, va: usize) -> Option<u64> {
+    // Root frame (mask off any ASID + attribute bits carried in the TTBR value).
+    let mut table_pa = (root_pa as u64 & PTE_ADDR_MASK) as usize;
+
+    // 4-level walk: level 0 indexes VA[47:39], … level 3 indexes VA[20:12].
+    for level in 0..4usize {
+        let shift = 39 - 9 * level;
+        let index = (va >> shift) & 0x1ff;
+        let pte = read_descriptor(table_pa, index)?;
+        if pte & PTE_VALID == 0 {
+            return None;
+        }
+        let table_or_page = pte & PTE_TABLE_OR_PAGE != 0;
+        if level < 3 {
+            if table_or_page {
+                // Table descriptor: descend.
+                table_pa = (pte & PTE_ADDR_MASK) as usize;
+                continue;
+            }
+            // Block (huge page) descriptor. A block is not permitted at level 0.
+            if level == 0 {
+                return None;
+            }
+        } else if !table_or_page {
+            // A level-3 descriptor with the type bit clear is reserved/invalid.
+            return None;
+        }
+
+        // Leaf reached (4 KiB page at level 3, 2 MiB block at level 2, 1 GiB block
+        // at level 1). Compose the output physical address for `va`.
+        let offset_mask = (1usize << shift) - 1;
+        let base = (pte & PTE_ADDR_MASK) as usize & !offset_mask;
+        return read_phys_word(base | (va & offset_mask));
+    }
+
+    // Unreachable: the level-3 iteration always returns.
+    None
+}
+
+/// Reads the 64-bit word at *user* virtual address `va` without ever faulting.
+///
+/// Walks the active user translation table (`TTBR0_EL1`). See [`read_word_via`].
+/// Allocation-free, lock-free, and safe from hard-IRQ context; an IRQ never
+/// switches `TTBR0_EL1`, so the interrupted user address space stays active.
+pub fn read_user_word_nofault(va: usize) -> Option<u64> {
+    read_word_via(ax_cpu::asm::read_user_page_table().as_usize(), va)
+}
+
+/// Reads the 64-bit word at *kernel* virtual address `va` without ever faulting.
+///
+/// Walks the kernel translation table (`TTBR1_EL1`) the same way as
+/// [`read_user_word_nofault`]. Used by the kernel FP unwinder so a corrupt or
+/// stale in-kernel frame pointer that slips past the range/alignment guards and
+/// points at an unmapped kernel VA yields `None` instead of an unrecoverable data
+/// abort (kernel panic) in hard-IRQ context — kernel stacks are mapped, so a
+/// legitimate frame still resolves and is read through the direct map.
+pub fn read_kernel_word_nofault(va: usize) -> Option<u64> {
+    read_word_via(ax_cpu::asm::read_kernel_page_table().as_usize(), va)
+}