From a44de89fd03680e88975ce504f241c159980a168 Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Mon, 20 Jul 2026 16:55:11 +0800 Subject: [PATCH 01/10] feat(cpufreq): RK3588 ondemand CPU DVFS with voltage calibration --- components/axcpu/src/aarch64/init.rs | 13 + drivers/ax-driver/Cargo.toml | 6 + drivers/ax-driver/src/lib.rs | 38 + drivers/ax-driver/src/soc/mod.rs | 4 +- drivers/ax-driver/src/soc/rockchip/cpufreq.rs | 809 +++++++++++++ drivers/ax-driver/src/soc/rockchip/mod.rs | 9 + .../ax-driver/src/soc/rockchip/pmic_i2c.rs | 719 ++++++++++++ .../ax-driver/src/soc/rockchip/pmic_spi.rs | 1044 +++++++++++++++++ drivers/ax-driver/src/soc/scmi.rs | 38 + os/StarryOS/kernel/src/entry.rs | 73 ++ os/arceos/modules/axtask/src/api.rs | 13 + os/arceos/modules/axtask/src/run_queue.rs | 23 +- 12 files changed, 2784 insertions(+), 5 deletions(-) create mode 100644 drivers/ax-driver/src/soc/rockchip/cpufreq.rs create mode 100644 drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs create mode 100644 drivers/ax-driver/src/soc/rockchip/pmic_spi.rs diff --git a/components/axcpu/src/aarch64/init.rs b/components/axcpu/src/aarch64/init.rs index 4fc82a3f15..1318adf905 100644 --- a/components/axcpu/src/aarch64/init.rs +++ b/components/axcpu/src/aarch64/init.rs @@ -118,6 +118,19 @@ pub fn init_trap() { #[cfg(feature = "uspace")] { CNTKCTL_EL1.modify(CNTKCTL_EL1::EL0VCTEN::TrappedNone + CNTKCTL_EL1::EL0PCTEN::TrappedNone); + // Start this CPU's free-running PMU cycle counter and let EL0 read it, so + // `PMCCNTR_EL0` yields real cycle counts at both EL1 and EL0. This is the + // exact-frequency oracle the DVFS calibration reads in-kernel and that + // `cpuprobe`'s `mhz_pmc` reads from userspace (both were 0/trapping before, + // because these registers were only ever set on the lazy perf_event_open + // path). Guarded on PMUv3 being present; a live system-wide `perf stat -e + // cycles` may momentarily reset/disable this shared counter, which is fine + // for a boot/idle calibration. + if crate::pmu::probe().is_some() { + crate::pmu::init_cpu(); + crate::pmu::cycles::configure(false, false); + crate::pmu::cycles::enable(); + } barrier::isb(barrier::SY); } unsafe extern "C" { diff --git a/drivers/ax-driver/Cargo.toml b/drivers/ax-driver/Cargo.toml index 40c319d2f2..dedb501ecc 100644 --- a/drivers/ax-driver/Cargo.toml +++ b/drivers/ax-driver/Cargo.toml @@ -136,6 +136,12 @@ rockchip-soc = [ "dep:rdif-reset", "dep:rockchip-soc", ] +# RK3588 fixed-OPP-at-boot CPU DVFS (voltage-free rung): raises the CPU cluster +# clocks via the SCMI seam at probe time. Board-only policy; keep it opt-in. +rk3588-cpufreq = [ + "rockchip-soc", + "dep:arm-scmi-rs", +] rockchip-pm = ["rockchip-soc", "dep:rdif-power", "dep:rockchip-pm"] list-pci-devices = ["pci"] pci-list-devices = ["list-pci-devices"] diff --git a/drivers/ax-driver/src/lib.rs b/drivers/ax-driver/src/lib.rs index af85932093..a35b6d0e9e 100644 --- a/drivers/ax-driver/src/lib.rs +++ b/drivers/ax-driver/src/lib.rs @@ -95,6 +95,44 @@ pub mod usb; #[cfg(virtio_dev)] pub mod virtio; +/// RK3588 CPU DVFS ondemand governor, exposed as a stable, arch-neutral entry +/// the kernel can drive from a periodic task without knowing the SoC specifics. +/// +/// The governor's *policy + apply* live in the (arch-specific) cpufreq driver, +/// but its *loop* — sleeping between samples and reading the per-CPU busy +/// counters — cannot live in this crate: ax-driver sits below ax-task/ax-hal in +/// the dependency graph, so spawning a task here would be a cyclic dependency. +/// The kernel therefore owns the loop and calls [`cpufreq::governor_poll`] each +/// tick. When the DVFS feature is off these are no-ops so callers stay generic. +pub mod cpufreq { + #[cfg(feature = "rk3588-cpufreq")] + pub use crate::soc::rockchip::cpufreq::{ + calibrate_cluster, calibrate_wanted, governor_period_ms, governor_poll, governor_wanted, + }; + + /// Feature-off stub: no governor, so the kernel never spawns its task. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn governor_wanted() -> bool { + false + } + /// Feature-off stub. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn governor_period_ms() -> u64 { + 100 + } + /// Feature-off stub. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn governor_poll(_busy: &[u64]) {} + /// Feature-off stub: no calibration. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn calibrate_wanted() -> bool { + false + } + /// Feature-off stub. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn calibrate_cluster(_cluster_idx: usize, _intended_cpu: usize) {} +} + #[cfg(feature = "pci")] pub use binding_info::PciIrqRequirement; pub use binding_info::{BindingInfo, BindingIrq, BindingIrqBinding, BindingIrqSource, FdtIrqSpec}; diff --git a/drivers/ax-driver/src/soc/mod.rs b/drivers/ax-driver/src/soc/mod.rs index fd40781641..7e45658c5b 100644 --- a/drivers/ax-driver/src/soc/mod.rs +++ b/drivers/ax-driver/src/soc/mod.rs @@ -15,8 +15,8 @@ #[cfg(feature = "pinctrl")] mod fixed_regulator; #[cfg(feature = "rockchip-soc")] -mod rockchip; -#[cfg(feature = "rockchip-dwmmc")] +pub(crate) mod rockchip; +#[cfg(any(feature = "rockchip-dwmmc", feature = "rk3588-cpufreq"))] pub mod scmi; #[cfg(feature = "starfive-soc")] mod starfive; diff --git a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs new file mode 100644 index 0000000000..60c2fbd579 --- /dev/null +++ b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs @@ -0,0 +1,809 @@ +// Copyright 2025 The Axvisor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RK3588 CPU DVFS: SCMI clocks + PMIC rail-voltage alignment + ondemand governor. +//! +//! The RK3588 CPU clock is voltage-coupled — an SCMI clock id selects a PVTPLL +//! ring whose *delivered* frequency tracks the core rail voltage — and the SCMI +//! interface is frequency-only. Exact DVFS therefore needs BOTH levers together: +//! the SCMI ring *and* the matching rail voltage. This driver does three things, +//! in order: +//! +//! 1. **Set each cluster clock** to its boot target over the board-proven SCMI +//! seam ([`set_and_verify`]). The three CPU domains and their SCMI clock ids +//! (ground truth `orangepi5plus.dts`: `cpu@0..300` → ``, `cpu@400/500` +//! → ``, `cpu@600/700` → ``): +//! +//! | cluster | SCMI clock id | boot target (MHz) | +//! |----------------|---------------|-------------------| +//! | A55 (little) | 0 | 1008 | +//! | A76 big pair 0 | 2 | 1200 | +//! | A76 big pair 1 | 3 | 1200 | +//! +//! 2. **Align each rail voltage to the OPP** ([`align_rail_voltages_to_opp`]). +//! Boot firmware leaves the rails high (~800 mV), so the coupled clock +//! overshoots the SCMI target until each rail is lowered to its OPP-nominal +//! (0.675 V for these OPPs on the standard SKU). Lowering cannot undervolt: the +//! coupled clock tracks the rail down in lockstep. NOTE the industrial +//! RK3588J/M SKU (selected by the `specification_serial_number` nvmem cell) +//! puts these OPPs at 0.75 V; the PMIC modules floor at 0.675 V, so gate the +//! nominal on the SKU cell before running this on a J/M part. +//! +//! 3. **Hand off to the ondemand governor** ([`governor_poll`]). Once both +//! CPU-rail PMIC buses are up, a dynamic governor scales each cluster's OPP to +//! match load (see the governor section at the end of this file). +//! +//! Registration and ordering: a `PostKernel` / `DEFAULT` rdrive probe, so the CRU + +//! SCMI providers (registered at `CLK` priority) are already live, and it runs +//! inside `devices::probe_all_devices()` — **before** `start_secondary_cpus()` — +//! so the A76 clusters are reclocked while no core is scheduled on them (the live +//! A55 id-0 switch is BL31's glitch-free path). It binds to the CPU nodes rather +//! than `arm,scmi-smc` (which the SCMI driver already owns; a second driver on +//! that node would never get an `on_probe`), and applies exactly once via a +//! one-shot guard because several `cpu@*` nodes match. + +use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; + +use fdt_edit::Phandle; +use log::{info, warn}; + +use crate::{probe::OnProbeError, register::ProbeFdt, soc::scmi}; + +/// SCMI clock id of the A55 (little) cluster — cpu0..3. +const A55_CLK_ID: u32 = 0; +/// SCMI clock ids of the two A76 (big) cluster pairs — cpu4/5 and cpu6/7. Both +/// must be set; they cover different core pairs. +const A76_CLK_IDS: [u32; 2] = [2, 3]; + +/// A55 target and hard ceiling: the top OPP still on the 816 MHz boot voltage +/// row. `set_clock_rate` must never be driven above this for the A55 cluster. +const A55_MAX_HZ: u64 = 1_008_000_000; +/// A76 target and hard ceiling: the top OPP still on the 816 MHz boot voltage +/// row. `set_clock_rate` must never be driven above this for an A76 cluster. +const A76_MAX_HZ: u64 = 1_200_000_000; + +/// One-shot guard: several `cpu@*` nodes match, but the reclock runs once. +static APPLIED: AtomicBool = AtomicBool::new(false); + +crate::model_register!( + name: "RK3588 CPU DVFS SCMI", + level: ProbeLevel::PostKernel, + priority: ProbePriority::DEFAULT, + probe_kinds: &[ + ProbeKind::Fdt { + compatibles: &["arm,cortex-a55", "arm,cortex-a76"], + on_probe: probe + } + ], +); + +fn probe(_probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { + // Several CPU nodes match this driver; only the first invocation reclocks. + if APPLIED.swap(true, Ordering::AcqRel) { + return Ok(()); + } + + // The `scmi::*` helpers ignore the phandle (single global agent); pass a + // dummy so we do not depend on parsing the node's clock specifier. + let phandle = Phandle::from(0u32); + + // Safety preflight (read-only): confirm this firmware actually services the + // CPU-cluster clocks before touching any of them. `describe_rates` changes + // no state, so this cannot hang or perturb the clocks; treat its result as + // accept/reject only. If any target id is rejected, leave every cluster at + // its boot rate and bail — there is no raw-CRU fallback here. + for id in [A55_CLK_ID, A76_CLK_IDS[0], A76_CLK_IDS[1]] { + if scmi::describe_rates(phandle, id).is_none() { + warn!( + "cpufreq: SCMI does not service CPU cluster clock id {id}; leaving all CPU \ + clusters at their boot rate (no DVFS applied)" + ); + return Ok(()); + } + } + + let a55_before = read_mhz(phandle, A55_CLK_ID); + if !set_and_verify(phandle, A55_CLK_ID, A55_MAX_HZ, A55_MAX_HZ) { + warn!("cpufreq: A55 reclock did not verify; stopping (A76 left at boot rate)"); + return Ok(()); + } + let a55_after = read_mhz(phandle, A55_CLK_ID); + + let a76_before = read_mhz(phandle, A76_CLK_IDS[0]); + for id in A76_CLK_IDS { + if !set_and_verify(phandle, id, A76_MAX_HZ, A76_MAX_HZ) { + warn!("cpufreq: A76 cluster clock id {id} reclock did not verify; stopping"); + return Ok(()); + } + } + let a76_after = read_mhz(phandle, A76_CLK_IDS[0]); + + info!("cpufreq: A55 {a55_before}->{a55_after}, A76 {a76_before}->{a76_after} MHz"); + + // The CPU clock is voltage-coupled (proven on-board: at a fixed SCMI clock the + // A76 runs ~1.19 GHz @675 mV but ~1.49 GHz @800 mV), so the clocks set above + // overshoot while each rail sits at its ~800 mV boot value. Lower each rail to + // its OPP nominal to pull the coupled clock onto the exact requested rate. Runs + // LAST — after the SCMI clock is confirmed at target — so the down-shift is the + // final step (matches Linux's reduce-freq-then-voltage order for + // down-transitions), and only on the full-success path above. + align_rail_voltages_to_opp(); + + // Hand off to the dynamic ondemand governor. It cannot + // live entirely in this crate — ax-driver sits *below* ax-task/ax-hal in the + // dependency graph (they pull ax-driver back in via axplat-dyn), so a + // task-spawning loop here would be a cyclic dep. Instead this driver exposes + // the pure policy+apply (`governor_poll`, see the governor section at the end + // of this file) and the kernel drives it from a periodic sleepable task. The + // voltage lever above armed `GOV_READY` iff both PMIC buses came up. + + Ok(()) +} + +/// Programs `clock_id` to `target`, but never above `ceiling` (the hard cap on +/// the boot voltage row), then verifies the platform actually applied it. +/// Returns `true` only when the read-back matches the request. +/// +/// `target == ceiling` for the boot targets; the clamp is defense in depth so a future +/// edit can never push a cluster past its boot-voltage-safe ceiling. A rejected +/// set or a deviating read-back is reported and returns `false` so the caller +/// stops rather than leaving a partial/phantom state — the board stays on +/// whatever rate the firmware last confirmed. +fn set_and_verify(phandle: Phandle, clock_id: u32, target: u64, ceiling: u64) -> bool { + if target > ceiling { + warn!( + "cpufreq: refusing to set clock id {clock_id} to {target} Hz (above boot-safe ceiling \ + {ceiling} Hz)" + ); + return false; + } + if scmi::set_clock_rate(phandle, clock_id, target).is_none() { + warn!("cpufreq: SCMI rejected clock id {clock_id} set to {target} Hz; left unchanged"); + return false; + } + match scmi::clock_rate(phandle, clock_id) { + Some(applied) if applied == target => true, + Some(applied) if applied > ceiling => { + warn!( + "cpufreq: clock id {clock_id} read back {applied} Hz ABOVE boot-safe ceiling \ + {ceiling} Hz (requested {target} Hz); stopping" + ); + false + } + Some(applied) => { + warn!( + "cpufreq: clock id {clock_id} read back {applied} Hz, requested {target} Hz; \ + stopping" + ); + false + } + None => { + warn!("cpufreq: could not read back clock id {clock_id} after set; stopping"); + false + } + } +} + +/// Current rate of `clock_id` in MHz, or 0 if it cannot be read (logging only). +fn read_mhz(phandle: Phandle, clock_id: u32) -> u64 { + scmi::clock_rate(phandle, clock_id).unwrap_or(0) / 1_000_000 +} + +// =========================================================================== +// CPU-rail voltage alignment (the voltage half of the coupled clock) +// =========================================================================== + +/// Master gate for the PMIC **writes**. While `false`, [`align_rail_voltages_to_opp`] +/// only *reads and logs* each rail's boot voltage (zero PMIC writes). Flipped to +/// `true` after the read-only board pass confirmed the A76 rails read the true +/// 800 mV boot voltage (i2c0 bring-up: ungate + reset + pinmux). Even with this +/// on, a rail is only lowered when its own read is trustworthy (see the A55 gate +/// below — its spi2/RK806 read is not up yet, so it is skipped). +const APPLY_RAIL_VOLTAGE: bool = true; + +/// OPP-nominal core voltage for the boot targets on the **standard** SKU: the +/// 675 mV row shared by the 816/1008/1200 MHz OPPs (board-confirmed from the +/// `cluster*-opp-table` `opp-microvolt`). The A55 1008 OPP and both A76 1200 OPPs +/// all sit here. +/// +/// NOTE: the industrial RK3588J/M SKU puts these OPPs at 750 mV (`opp-j-m-*`). The +/// PMIC modules floor at 675 mV, so a 675 mV target would *under*-volt a J/M part. +/// This board is the standard SKU; if this driver is ever run on a J/M board, gate +/// this constant on the `specification_serial_number` nvmem SKU cell first. +const A76_NOMINAL_UV: u32 = 675_000; +const A55_NOMINAL_UV: u32 = 675_000; + +/// One-shot A55 MOSI/write diagnostic (user-authorized). The RK806 read path is +/// dead (returns a bogus 0x00), so we cannot validate an A55 write by read-back. +/// This force-writes DCDC2 to the bounded-safe A55 OPP nominal and relies on +/// cpuprobe observing whether the A55 frequency drops — the only way to learn if +/// MOSI/writes physically reach the RK806 when reads do not. The write is clamped +/// to [675 mV, 800 mV] (the A55 boot-safe row) inside the PMIC module. +const A55_FORCE_WRITE_TEST: bool = true; + +/// Read (and, once validated, lower) the three CPU-cluster rails to their OPP +/// nominal so the voltage-coupled clock lands on the exact requested frequency. +/// +/// Order matters: the A76 clusters are reclocked before `start_secondary_cpus()` +/// so no core is scheduled on them — their voltage is lowered first. The A55 rail +/// feeds the live boot core, so it is lowered last, and only via the stepped path +/// (the voltage-coupled clock tracks the rail down with no undervolt transient). +fn align_rail_voltages_to_opp() { + use super::{pmic_i2c, pmic_spi}; + + // --- A76 big0/big1 rails: RK8602 @0x42 / RK8603 @0x43 over I2C bus0 --- + let a76_ok = pmic_i2c::init(); + if a76_ok { + for (name, chip) in [ + ("big0", pmic_i2c::RK8602_BIG0_ADDR), + ("big1", pmic_i2c::RK8603_BIG1_ADDR), + ] { + match pmic_i2c::get_uv(chip) { + Some(uv) => info!("cpufreq: A76 {name} rail boot voltage = {uv} uV"), + None => warn!("cpufreq: A76 {name} rail voltage read failed"), + } + } + } else { + warn!("cpufreq: A76 PMIC (I2C) init failed; A76 left at boot voltage"); + } + + // --- A55 (little) rail: RK806 DCDC2 over SPI2 --- + let a55_ok = pmic_spi::init(); + if a55_ok { + match pmic_spi::get_uv() { + Some(uv) => info!("cpufreq: A55 rail boot voltage = {uv} uV"), + None => warn!("cpufreq: A55 rail voltage read failed"), + } + } else { + warn!("cpufreq: A55 PMIC (SPI) init failed; A55 left at boot voltage"); + } + + if !APPLY_RAIL_VOLTAGE { + info!("cpufreq: rail-voltage alignment is READ-ONLY this build (no PMIC writes)"); + return; + } + + // --- Apply: stepped, down-only, read-back-verified lower to OPP nominal. --- + if a76_ok { + let b0 = pmic_i2c::set_uv_stepped(pmic_i2c::RK8602_BIG0_ADDR, A76_NOMINAL_UV); + let b1 = pmic_i2c::set_uv_stepped(pmic_i2c::RK8603_BIG1_ADDR, A76_NOMINAL_UV); + info!("cpufreq: A76 rails -> {A76_NOMINAL_UV} uV (big0 ok={b0}, big1 ok={b1})"); + } + // A55 (spi2/RK806): only lower it if the current-voltage read is trustworthy. + // `set_uv_stepped` reads the rail before stepping down, so we must never lower a + // rail we cannot read. The spi2/RK806 read path is not up yet (it returns a bogus + // 0x00 == 500 mV), so skip the A55 write until that bring-up completes rather than + // act on a false reading. A real A55 boot voltage is in [675 mV, 950 mV] per the + // RK806 DCDC2 range and the cluster0 OPP table. + match if a55_ok { pmic_spi::get_uv() } else { None } { + Some(v) if (675_000..=950_000).contains(&v) => { + let a55 = pmic_spi::set_uv_stepped(A55_NOMINAL_UV); + info!("cpufreq: A55 rail {v} -> {A55_NOMINAL_UV} uV (ok={a55})"); + } + other => warn!( + "cpufreq: A55 boot voltage not trustworthy ({other:?} uV); skipping A55 write until \ + spi2/RK806 bring-up completes (A76 unaffected)" + ), + } + + // A55 MOSI/write diagnostic: force-write DCDC2 to the bounded-safe nominal and + // let cpuprobe reveal whether writes reach the RK806 even though reads don't. + if A55_FORCE_WRITE_TEST && a55_ok { + let ok = pmic_spi::force_write_dcdc2(A55_NOMINAL_UV); + info!( + "cpufreq: A55 MOSI/write test -> {A55_NOMINAL_UV} uV (write xfer_ok={ok}); watch A55 \ + cpuprobe (req=0..3) for a freq drop if the write reached" + ); + } + + // Arm the dynamic governor only if both PMIC buses came up, so it never tries + // to move a rail it cannot drive. If either failed, every cluster stays on the + // boot OPP the SCMI reclock already set (safe: that is the fixed-boot state). + if GOVERNOR_ENABLE && a76_ok && a55_ok { + GOV_READY.store(true, Ordering::Release); + info!("cpufreq: ondemand governor armed (both PMIC buses up)"); + } else if GOVERNOR_ENABLE { + warn!( + "cpufreq: ondemand governor NOT armed (a76_pmic={a76_ok}, a55_pmic={a55_ok}); \ + clusters stay on boot OPP" + ); + } +} + +// =========================================================================== +// Dynamic ondemand governor +// =========================================================================== +// +// The voltage lever above pins each cluster at a single boot OPP. This governor +// makes DVFS *dynamic*: it samples per-CPU busy time and moves each cluster's +// OPP up and down to track load, the way Linux's `ondemand`/`schedutil` do. +// +// Split by cost, exactly like Linux: the *accounting* is a cheap per-CPU counter +// bumped in the scheduler tick (`ax_task::cpu_busy_ticks`, fed by the non-idle +// branch of `scheduler_timer_tick`), but the *apply* is slow and SLEEPS — an +// SCMI clock set is an SMC into BL31, and the paired PMIC write is an I2C/SPI +// transaction with a millisecond voltage ramp. Neither may run in the tick +// handler, so the decision+apply live in a periodic sleepable kernel task. This +// is exactly why Linux's old ondemand used a deferred timer and schedutil kicks +// a kthread rather than reprogramming the OPP inline in the tick. + +/// Master gate for the dynamic governor. When `false`, each cluster simply stays +/// on the fixed boot OPP the voltage alignment left it on. +const GOVERNOR_ENABLE: bool = true; + +/// An operating performance point: the SCMI ring target to program, the rail +/// voltage to pair with it, and the frequency that combination actually delivers +/// (`mhz`, board-measured — see the calibration section). Because the PVTPLL is +/// voltage-coupled, the delivered `mhz` is generally NOT the `ring_khz`; the +/// governor reports `mhz`. +#[derive(Clone, Copy)] +struct Opp { + ring_khz: u32, + uv: u32, + mhz: u32, +} + +/// A76 (big) OPP ladder, low→high, from the on-board calibration sweep. The clock +/// is voltage-coupled, so this ladder is a HYBRID: below the 675 mV exact point it +/// scales the SCMI ring (408/816/1200 @ 675 mV land on target); above it, it holds +/// the ring at 1200 and raises the *voltage* — the delivered freq climbs while +/// staying over-volted (each rung's voltage exceeds the delivered freq's DT +/// nominal, so never an undervolt). Scaling the ring instead (e.g. ring 1608 @ +/// 762.5 mV) over-delivers ~1733 MHz = ~125 mV of undervolt (measured), so it is +/// avoided. Top rung 1725 MHz @ 925 mV is the calibration sweep's safe maximum +/// (over-volted ~110 mV vs the delivered freq's DT nominal); board-validated +/// all-core (threads=8) with no PSU brownout. +const A76_OPPS: &[Opp] = &[ + Opp { + ring_khz: 408_000, + uv: 675_000, + mhz: 408, + }, + Opp { + ring_khz: 816_000, + uv: 675_000, + mhz: 816, + }, + Opp { + ring_khz: 1_200_000, + uv: 675_000, + mhz: 1189, + }, + Opp { + ring_khz: 1_200_000, + uv: 725_000, + mhz: 1318, + }, + Opp { + ring_khz: 1_200_000, + uv: 800_000, + mhz: 1491, + }, + Opp { + ring_khz: 1_200_000, + uv: 850_000, + mhz: 1592, + }, + Opp { + ring_khz: 1_200_000, + uv: 925_000, + mhz: 1725, + }, +]; + +/// A55 (little) OPP ladder, low→high, same hybrid rationale: ring-scaled below the +/// 675 mV point, then ring 1008 with rising voltage. Top rung 1523 MHz @ 950 mV +/// (the RK806 force-write ceiling), the sweep's safe maximum for the little cluster. +const A55_OPPS: &[Opp] = &[ + Opp { + ring_khz: 408_000, + uv: 675_000, + mhz: 408, + }, + Opp { + ring_khz: 816_000, + uv: 675_000, + mhz: 816, + }, + Opp { + ring_khz: 1_008_000, + uv: 675_000, + mhz: 1021, + }, + Opp { + ring_khz: 1_008_000, + uv: 762_500, + mhz: 1212, + }, + Opp { + ring_khz: 1_008_000, + uv: 800_000, + mhz: 1285, + }, + Opp { + ring_khz: 1_008_000, + uv: 850_000, + mhz: 1372, + }, + Opp { + ring_khz: 1_008_000, + uv: 950_000, + mhz: 1523, + }, +]; + +/// Index into each ladder of the boot OPP the voltage lever leaves the cluster on: +/// A55 1008 MHz and A76 1200 MHz are both element 2 (the 675 mV rung). The governor +/// starts tracking here so its first move is relative to the known boot state; from +/// idle it decays down the ring-scaled rungs and under load climbs the voltage ones. +const BOOT_OPP_IDX: usize = 2; + +/// The three DVFS domains (one little cluster, two big pairs). +#[derive(Clone, Copy)] +enum Cluster { + A55, + Big0, + Big1, +} + +impl Cluster { + fn opps(self) -> &'static [Opp] { + match self { + Cluster::A55 => A55_OPPS, + _ => A76_OPPS, + } + } + + /// SCMI clock id feeding this domain's PVTPLL ring. + fn clock_id(self) -> u32 { + match self { + Cluster::A55 => A55_CLK_ID, + Cluster::Big0 => A76_CLK_IDS[0], + Cluster::Big1 => A76_CLK_IDS[1], + } + } + + /// CPUs whose busy time this domain aggregates (RK3588 topology: cpu0-3 A55, + /// cpu4-5 big pair 0, cpu6-7 big pair 1). + fn cpus(self) -> core::ops::Range { + match self { + Cluster::A55 => 0..4, + Cluster::Big0 => 4..6, + Cluster::Big1 => 6..8, + } + } + + fn name(self) -> &'static str { + match self { + Cluster::A55 => "A55", + Cluster::Big0 => "A76b0", + Cluster::Big1 => "A76b1", + } + } + + /// Set this domain's rail to `uv`. A76 uses the read-back-verified I2C + /// regulator; A55 uses the bounded force-write (its RK806 read is a + /// scope-wall, but writes reach the chip — proven by the rail-alignment freq drop). + /// Both PMIC helpers clamp to their rail's safe envelope internally. + fn set_voltage(self, uv: u32) -> bool { + use super::{pmic_i2c, pmic_spi}; + match self { + Cluster::A55 => pmic_spi::force_write_dcdc2(uv), + Cluster::Big0 => pmic_i2c::set_uv(pmic_i2c::RK8602_BIG0_ADDR, uv), + Cluster::Big1 => pmic_i2c::set_uv(pmic_i2c::RK8603_BIG1_ADDR, uv), + } + } +} + +/// Apply an OPP to a domain as a matched (voltage, frequency) pair, ordered so +/// the voltage-coupled clock never overshoots its rail: +/// - going UP: raise voltage first, then the SCMI ring (clock follows up); +/// - going DOWN: lower the SCMI ring first, then voltage (clock follows down). +fn apply_opp(cluster: Cluster, opp: Opp, going_up: bool) { + let phandle = Phandle::from(0u32); + let hz = opp.ring_khz as u64 * 1_000; + if going_up { + cluster.set_voltage(opp.uv); + scmi::set_clock_rate(phandle, cluster.clock_id(), hz); + } else { + scmi::set_clock_rate(phandle, cluster.clock_id(), hz); + cluster.set_voltage(opp.uv); + } +} + +/// Period, in ms, the kernel governor task sleeps between [`governor_poll`]s. +const GOV_PERIOD_MS: u64 = 100; +/// Busy% at or above which a domain jumps straight to its top OPP (ondemand's +/// signature fast attack: respond to a load spike in one step). +const UP_THRESHOLD_PCT: u64 = 80; +/// Busy% below which a domain steps down one OPP (slow decay: shed frequency +/// gradually so a brief idle dip does not collapse a still-busy workload). +const DOWN_THRESHOLD_PCT: u64 = 30; + +/// Set once the voltage lever confirmed both PMIC buses are up. Until then the +/// governor must not move any rail (see `align_rail_voltages_to_opp`). +static GOV_READY: AtomicBool = AtomicBool::new(false); + +/// Per-CPU busy-tick value from the previous poll (RK3588 has 8 cores). Only the +/// single governor task ever touches these, so `Relaxed` is sufficient. +static LAST_BUSY: [AtomicU64; 8] = [const { AtomicU64::new(0) }; 8]; + +/// Per-cluster current OPP index (A55, big0, big1), starting on the boot OPP the +/// voltage lever pinned. +static IDX: [AtomicUsize; 3] = [const { AtomicUsize::new(BOOT_OPP_IDX) }; 3]; + +/// Cleared until the first [`governor_poll`] has recorded a busy baseline. The +/// first call must only prime `LAST_BUSY`, not decide: its delta is measured +/// from zero and would otherwise fold in every busy tick accumulated since boot, +/// spuriously pegging every cluster to its top OPP for one window. +static PRIMED: AtomicBool = AtomicBool::new(false); + +/// Whether the dynamic governor should run: enabled at compile time *and* armed +/// by the voltage lever (both PMIC buses up). The kernel checks this once before +/// spawning its periodic governor task, so a failed PMIC bring-up leaves every +/// cluster safely on the boot OPP instead of being scaled with no voltage lever. +pub fn governor_wanted() -> bool { + GOVERNOR_ENABLE && GOV_READY.load(Ordering::Acquire) +} + +/// Period, in milliseconds, the kernel governor task should sleep between calls +/// to [`governor_poll`]. +pub fn governor_period_ms() -> u64 { + GOV_PERIOD_MS +} + +/// One ondemand iteration, called periodically by the kernel governor task with +/// a fresh snapshot of every CPU's cumulative busy-tick counter +/// (`ax_task::cpu_busy_ticks`). Scores each core's busy% over the last window and +/// moves each cluster's OPP from its busiest core: any saturated core jumps the +/// cluster to its top OPP, and the cluster sheds a step only when all its cores +/// are near-idle. Applies via SCMI+PMIC. Pure w.r.t. the task runtime — it +/// neither sleeps nor spawns — so this crate needs no dependency on +/// ax-task/ax-hal (which would be a cyclic dep through axplat-dyn). +/// +/// `busy[i]` is CPU `i`'s counter; indices past the slice, or offline CPUs whose +/// counter never advances, simply read as idle — conservative (never over-scales). +pub fn governor_poll(busy: &[u64]) { + if !governor_wanted() { + return; + } + + // The task sleeps `GOV_PERIOD_MS`, so ~`GOV_PERIOD_MS / 10` scheduler ticks + // (10 ms/tick, TICKS_PER_SEC = 100) elapse per window. Using the nominal + // window needs no clock here; a late wake only under-reports load (busy% is + // clamped below), which is safe — it can never spuriously over-scale. + const WINDOW_TICKS: u64 = if GOV_PERIOD_MS / 10 == 0 { + 1 + } else { + GOV_PERIOD_MS / 10 + }; + + // First call only establishes the baseline (see `PRIMED`); still walk every + // cluster below so all `LAST_BUSY` entries are seeded, but make no OPP change. + let priming = !PRIMED.swap(true, Ordering::Relaxed); + + for (ci, &cluster) in [Cluster::A55, Cluster::Big0, Cluster::Big1] + .iter() + .enumerate() + { + // Score each core in the cluster individually this window. A cluster + // shares ONE clock, so a single saturated core is reason to raise the + // whole cluster — this matches Linux schedutil/ondemand, which drive a + // frequency domain from its busiest CPU. Averaging instead (an earlier + // bug) buried one CPU-bound thread among its idle siblings: a single + // thread on the 2-core A76 pair only reads 50%, below the up-threshold, + // so the cluster never boosted. Each CPU belongs to exactly one cluster, + // so every LAST_BUSY entry is refreshed exactly once per poll. + let mut any_core_high = false; // some core wants the top OPP + let mut all_cores_low = true; // every core is near-idle → shed one step + let mut peak_pct = 0u64; // busiest core, for the log line + let mut n = 0u64; + for cpu in cluster.cpus() { + let now = busy.get(cpu).copied().unwrap_or(0); + let last = LAST_BUSY[cpu].swap(now, Ordering::Relaxed); + // Per-core busy% = busy_ticks / window_ticks (one core), clamped. + let pct = ((now.saturating_sub(last) * 100) / WINDOW_TICKS).min(100); + if pct >= UP_THRESHOLD_PCT { + any_core_high = true; + } + if pct >= DOWN_THRESHOLD_PCT { + all_cores_low = false; + } + if pct > peak_pct { + peak_pct = pct; + } + n += 1; + } + if n == 0 || priming { + continue; + } + + let opps = cluster.opps(); + let cur = IDX[ci].load(Ordering::Relaxed); + let new = if any_core_high { + opps.len() - 1 // fast attack: jump straight to the top OPP + } else if all_cores_low && cur > 0 { + cur - 1 // slow decay: shed one step only when the whole cluster is idle + } else { + cur + }; + + if new != cur { + apply_opp(cluster, opps[new], new > cur); + IDX[ci].store(new, Ordering::Relaxed); + info!( + "gov: {} peak={}% opp {}->{} = {} MHz @ {} mV", + cluster.name(), + peak_pct, + cur, + new, + opps[new].mhz, + opps[new].uv / 1_000 + ); + } + } +} + +// =========================================================================== +// OPP calibration sweep (gated, one-shot) +// =========================================================================== +// +// The PVTPLL clock is voltage-coupled, so the delivered frequency for a given +// SCMI ring drifts with rail voltage, and the drift grows at the higher OPPs +// (ring=1608 @ 762.5 mV measured near ~1733 MHz on-board). To use the +// 1416/1608/1800 rungs SAFELY we must know the *actual* delivered frequency at +// each (ring, voltage); this sweep measures it directly via the PMU cycle +// counter. It runs once, gated by `CALIBRATE`, from early `init()` (before the +// console tty handoff) so its `CAL` log lines reach the serial console — the +// governor's own transition logs do not, because they fire post-handoff. + +/// One-shot gate: when true, `init()` runs [`calibrate_cluster`] per cluster +/// (governor NOT spawned) and the board logs a `CAL` grid; leave false for +/// production. Requires the PMU cycle counter enabled at boot (axcpu `init_trap`). +const CALIBRATE: bool = false; + +/// Sweep points `(rail_uV, ring_kHz)`. Round 1 showed the delivered frequency is +/// dominated by voltage and that at any DT (ring=F, V_nom(F)) pair the delivery +/// *over*-shoots F (undervolt). The safe lever is instead a FIXED low ring with a +/// rising voltage: the delivered freq climbs but stays over-volted. So round 2 +/// maps ring 1200 across the full voltage range (plus two higher-ring cross-checks +/// to see where the ring stops mattering). Every point keeps V >= V_nom(ring), so +/// no measured point undervolts a live core; list is voltage-non-decreasing. +const CAL_A76: &[(u32, u32)] = &[ + (675_000, 1_200_000), + (725_000, 1_200_000), + (762_500, 1_200_000), + (800_000, 1_200_000), + (850_000, 1_200_000), + (925_000, 1_200_000), + (850_000, 1_416_000), // cross-check: does a higher ring beat ring 1200 at 850? + (925_000, 1_608_000), // cross-check at 925 +]; + +/// A55: fixed ring 1008 across the voltage range (RK806 force-write caps at 950 mV). +const CAL_A55: &[(u32, u32)] = &[ + (675_000, 1_008_000), + (712_500, 1_008_000), + (762_500, 1_008_000), + (800_000, 1_008_000), + (850_000, 1_008_000), + (950_000, 1_008_000), + (850_000, 1_416_000), // cross-check + (950_000, 1_608_000), // cross-check +]; + +/// Whether to run the calibration sweep this boot (compile gate + PMIC armed). +pub fn calibrate_wanted() -> bool { + CALIBRATE && GOV_READY.load(Ordering::Acquire) +} + +#[inline] +fn rd_pmccntr() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, pmccntr_el0", out(reg) v) }; + v +} +#[inline] +fn rd_cntvct() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, cntvct_el0", out(reg) v) }; + v +} +#[inline] +fn rd_cntfrq() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) v) }; + v +} +#[inline] +fn rd_mpidr() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) v) }; + v +} + +/// Busy-wait `ms` milliseconds against the fixed-rate `CNTVCT` clock. +fn cal_delay_ms(ms: u64) { + let frq = rd_cntfrq().max(1); + let start = rd_cntvct(); + let ticks = frq * ms / 1000; + while rd_cntvct().wrapping_sub(start) < ticks { + core::hint::spin_loop(); + } +} + +/// Measure the CURRENT core's frequency (MHz) by counting CPU cycles +/// (`PMCCNTR_EL0`) over a fixed ~60 ms window timed by `CNTVCT_EL0`. The counter +/// is 32-bit (no `PMCR_EL0.LC`), which cannot wrap over this window. +fn measure_mhz() -> u32 { + let frq = rd_cntfrq().max(1); + let window = frq * 60 / 1000; // 60 ms in cntvct ticks + let t0 = rd_cntvct(); + let c0 = rd_pmccntr() as u32; + while rd_cntvct().wrapping_sub(t0) < window { + core::hint::spin_loop(); + } + let t1 = rd_cntvct(); + let c1 = rd_pmccntr() as u32; + let dvct = t1.wrapping_sub(t0).max(1); + let dcyc = c1.wrapping_sub(c0) as u64; // 32-bit wrap-safe + ((dcyc * frq) / (dvct * 1_000_000)) as u32 +} + +/// Run the (voltage x ring) calibration sweep for one cluster ON THE CURRENT +/// CORE, logging the delivered frequency at each point. `cluster_idx`: 0=A55, +/// 1=A76 big0, 2=A76 big1. The caller must have pinned this task onto a core in +/// the target cluster (the logged `MPIDR` aff1 lets you confirm it). Restores a +/// safe boot OPP for the cluster on exit. PMIC access here is pure polling, so it +/// is safe on a non-boot core. +pub fn calibrate_cluster(cluster_idx: usize, intended_cpu: usize) { + let (cluster, points, restore_khz) = match cluster_idx { + 0 => (Cluster::A55, CAL_A55, 1_008_000u32), + 1 => (Cluster::Big0, CAL_A76, 1_200_000u32), + _ => (Cluster::Big1, CAL_A76, 1_200_000u32), + }; + let mpidr = rd_mpidr(); + info!( + "CAL begin cl={} cpu={} mpidr_aff1={} aff0={}", + cluster.name(), + intended_cpu, + (mpidr >> 8) & 0xff, + mpidr & 0xff + ); + let phandle = Phandle::from(0u32); + for &(uv, khz) in points { + // Voltage first (points are voltage-non-decreasing, so this only ever + // over-volts the current ring = safe), then the SCMI ring. + let vok = cluster.set_voltage(uv); + scmi::set_clock_rate(phandle, cluster.clock_id(), khz as u64 * 1_000); + cal_delay_ms(25); + let f = measure_mhz(); + info!( + "CAL cl={} volt={}uV ring={}MHz volt_ok={} => {}MHz", + cluster.name(), + uv, + khz / 1_000, + vok, + f + ); + } + // Restore: ring down first, then voltage down (safe order). + scmi::set_clock_rate(phandle, cluster.clock_id(), restore_khz as u64 * 1_000); + cluster.set_voltage(675_000); + info!( + "CAL end cl={} restored {}MHz@675mV", + cluster.name(), + restore_khz / 1_000 + ); +} diff --git a/drivers/ax-driver/src/soc/rockchip/mod.rs b/drivers/ax-driver/src/soc/rockchip/mod.rs index a90783299d..f31b7c2003 100644 --- a/drivers/ax-driver/src/soc/rockchip/mod.rs +++ b/drivers/ax-driver/src/soc/rockchip/mod.rs @@ -12,6 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(feature = "rk3588-cpufreq")] +pub(crate) mod cpufreq; + +#[cfg(feature = "rk3588-cpufreq")] +mod pmic_i2c; + +#[cfg(feature = "rk3588-cpufreq")] +mod pmic_spi; + #[cfg(feature = "rockchip-soc")] mod cru; diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs new file mode 100644 index 0000000000..79c20b9b17 --- /dev/null +++ b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs @@ -0,0 +1,719 @@ +// Copyright 2025 The Axvisor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RK3588 A76 CPU-rail regulator lever (DVFS Phase 2) — RK8602/RK8603 over the +//! `rk3x_i2c` PMIC bus. +//! +//! This is the **voltage half** of the RK3588 CPU-DVFS fix. Phase 1a raises the +//! A76 cluster clocks via SCMI/BL31, but RK3588's CPU clock is voltage-coupled, +//! so the clock overshoots the request while the rail stays at its 800 mV boot +//! value. Lowering each A76 rail to its OPP-nominal 675 mV pulls the coupled +//! clock down to the exact requested rate. This module provides the read/write +//! API; `cpufreq.rs` owns when it is called. +//! +//! # Components +//! +//! * A minimal **polling** `rk3x_i2c` master for `i2c@fd880000` (RK3588 "bus0", +//! the PMIC bus). No IRQs: it polls the raw IPD status. u-boot has already +//! clocked and used this controller (it programs these same PMICs at boot), +//! so [`Rk3xI2c::init_controller`] preserves u-boot's `CLKDIV` (a proven-good +//! SCL rate) and only clears state. Register map + transaction flow modelled +//! on mainline u-boot `drivers/i2c/rk_i2c.c` and Linux +//! `drivers/i2c/busses/i2c-rk3x.c`. +//! * A **RK8602/RK8603** regulator API (`get_uv` / `set_uv` / `set_uv_stepped`) +//! for the two A76 rails (RK8602 @ `0x42` = big0/cpu4-5, RK8603 @ `0x43` = +//! big1/cpu6-7). +//! +//! # RK8602/RK8603 VSEL register (finding) +//! +//! Linux handles RK8602/RK8603 in `drivers/regulator/fan53555.c` +//! (`RK8602_VENDOR_ROCKCHIP`). It defines two voltage registers, +//! `RK8602_VSEL0 = 0x06` and `RK8602_VSEL1 = 0x07`, and selects the **active +//! runtime** register (`di->vol_reg`) from the `sleep_vsel_id` (DT +//! `fcs,suspend-voltage-selector`): `sleep_vsel_id == 1` makes VSEL1 (`0x07`) +//! the *sleep* register and **VSEL0 (`0x06`) the active** one. The Orange Pi +//! 5 Plus RK8602/8603 nodes use `fcs,suspend-voltage-selector = <1>`, so the +//! board's live VSEL is `0x06` — matching the board-confirmed boot value +//! (`0x06 == 0x30 == 800 mV`). We therefore write/read **reg `0x06`**. +//! +//! The voltage code is a full 8-bit field (`RK8602_NVOLTAGES = 160`, +//! `vsel_mask = 0xff`) — no enable/mode bit shares reg `0x06` (the buck-enable +//! `VSEL_BUCK_EN` bit lives in the legacy FAN53555 registers `0x00/0x01`, left +//! untouched — u-boot already enabled the rail). Encoding matches the board: +//! `di->vsel_min = 500000`, `di->vsel_step = 6250` → `V = 500000 + code·6250` +//! µV, i.e. `0x1c → 675 mV`, `0x30 → 800 mV`. + +use core::time::Duration; + +use ax_kspin::SpinNoIrq as Mutex; +use log::{info, warn}; +use mmio_api::{MmioAddr, MmioRaw}; +use rdif_pinctrl::PinctrlDevice; + +use crate::mmio::iomap; + +// --------------------------------------------------------------------------- +// rk3x_i2c register map (offsets from the controller base) +// --------------------------------------------------------------------------- + +const REG_CON: usize = 0x00; // control +const REG_CLKDIV: usize = 0x04; // clock divider +const REG_MRXADDR: usize = 0x08; // master-rx slave address +const REG_MRXRADDR: usize = 0x0c; // master-rx slave register address +const REG_MTXCNT: usize = 0x10; // master-tx byte count +const REG_MRXCNT: usize = 0x14; // master-rx byte count +const REG_IEN: usize = 0x18; // interrupt enable +const REG_IPD: usize = 0x1c; // interrupt pending (raw status we poll) +const TXDATA_BASE: usize = 0x100; // tx FIFO word 0 +const RXDATA_BASE: usize = 0x200; // rx FIFO word 0 + +// REG_CON bits +const CON_EN: u32 = 1 << 0; +const CON_START: u32 = 1 << 3; +const CON_STOP: u32 = 1 << 4; +const CON_LASTACK: u32 = 1 << 5; // NACK after the last received byte + +// REG_CON transfer modes (shifted into bits [2:1] by `con_mod`) +const MODE_TX: u32 = 0; // write: address + data from tx FIFO +const MODE_TRX: u32 = 1; // combined: write reg address, then read + +const fn con_mod(mode: u32) -> u32 { + mode << 1 +} + +// REG_MRXADDR/REG_MRXRADDR: BIT(24 + n) marks address byte n valid. We only +// ever send one address byte and one register byte, so byte 0 valid = BIT(24). +const MRXADDR_VALID0: u32 = 1 << 24; + +// REG_IPD/REG_IEN interrupt bits +const INT_MBTF: u32 = 1 << 2; // master byte-transmit finished +const INT_MBRF: u32 = 1 << 3; // master byte-receive finished +const INT_START: u32 = 1 << 4; // START generated +const INT_STOP: u32 = 1 << 5; // STOP generated +const INT_NAKRCV: u32 = 1 << 6; // NACK received +const INT_ALL: u32 = 0x7f; // all pending bits (write-1-to-clear) + +/// RK3588 PMIC I2C controller ("bus0", `i2c@fd880000`). +const RK3588_I2C0_BASE: usize = 0xfd88_0000; +const RK3588_I2C0_SIZE: usize = 0x1000; + +// --- i2c0 clock ungate (PMU CRU) -------------------------------------------- +// +// i2c0 lives in the PMU CRU (`pmucru: clock-controller@fd7f0000` = main CRU +// 0xfd7c0000 + 0x30000). u-boot programmed the PMICs over i2c0 at boot but +// StarryOS leaves **CLK_I2C0 gated**: PCLK is still on (registers readable, e.g. +// CLKDIV reads back), yet the controller cannot generate SCL, so any START +// times out. `init` ungates it before the first transaction. +// +// Register/bit ground truth (cross-checked, identical in both): +// * mainline `drivers/clk/rockchip/clk-rk3588.c`: +// GATE(PCLK_I2C0, ..., RK3588_PMU_CLKGATE_CON(2), 1) +// COMPOSITE_NODIV(CLK_I2C0, ..., RK3588_PMU_CLKGATE_CON(2), 2) +// * `rockchip-soc` CRU crate gate.rs: `PCLK_I2C0 => (2, 1)`, `CLK_I2C0 => (2, 2)`. +// * PMU_CLKGATE_CON(x) = 0x800 + x*4 within the PMU CRU → CON(2) = 0x808. +/// PMU CRU physical base (`clock-controller@fd7f0000`). +const RK3588_PMU_CRU_BASE: usize = 0xfd7f_0000; +const RK3588_PMU_CRU_SIZE: usize = 0x1000; +/// `PMU_CLKGATE_CON(2)` offset within the PMU CRU (holds the i2c0 gates). +const PMU_CLKGATE_CON2: usize = 0x800 + 2 * 4; +/// Gate bits to clear: PCLK_I2C0 (bit 1) + CLK_I2C0 (bit 2). +const I2C0_GATE_BITS: u32 = (1 << 1) | (1 << 2); + +// i2c0 soft-reset is in the same PMU CRU. Ungating the clock alone was not +// enough on-board (registers readable, but START still timed out): the i2c0 +// core is held in **soft-reset**. SRST_P_I2C0 (PCLK reset) = bit 1 and +// SRST_I2C0 (functional/core reset) = bit 2 of PMU_SOFTRST_CON(2), from mainline +// `drivers/clk/rockchip/rst-rk3588.c:776-777` +// (`RK3588_PMU1CRU_RESET_OFFSET(SRST_P_I2C0, 2, 1)` / `(SRST_I2C0, 2, 2)`) — +// same bit layout as the gate. The functional reset (bit 2) matches the +// symptom: PCLK regs read (SRST_P deasserted) but the SCL state machine is held. +// We only DE-ASSERT (write 0, masked), never assert, so u-boot's preserved +// CLKDIV in the PCLK domain is not cleared. +/// `PMU_SOFTRST_CON(2)` offset within the PMU CRU (0xa00 + 2*4). +const PMU_SOFTRST_CON2: usize = 0xa00 + 2 * 4; +/// Reset bits to release: SRST_P_I2C0 (bit 1) + SRST_I2C0 (bit 2). +const I2C0_RESET_BITS: u32 = (1 << 1) | (1 << 2); + +/// Poll budget for a single IPD event, at 1 µs granularity — ~100 ms, matching +/// u-boot's `I2C_TIMEOUT_MS`. A PMIC register access completes in tens of µs; +/// this only bounds a wedged bus. +const I2C_POLL_MAX: u32 = 100_000; + +// --------------------------------------------------------------------------- +// RK8602/RK8603 regulator constants +// --------------------------------------------------------------------------- + +/// RK8602 @ I2C `0x42` — A76 big0 rail (`vdd_cpu_big0_s0`, cpu4-5). +pub const RK8602_BIG0_ADDR: u8 = 0x42; +/// RK8603 @ I2C `0x43` — A76 big1 rail (`vdd_cpu_big1_s0`, cpu6-7). +pub const RK8603_BIG1_ADDR: u8 = 0x43; + +/// Active runtime VSEL register on this board (RK8602_VSEL0). See module docs. +const VSEL_REG: u8 = 0x06; +/// 8-bit voltage-code mask (`RK8602_NVOLTAGES = 160`, `vsel_mask = 0xff`). +const VSEL_MASK: u8 = 0xff; +/// Voltage encoding: `V = VSEL_BASE_UV + code · VSEL_STEP_UV` (µV). +const VSEL_BASE_UV: u32 = 500_000; +const VSEL_STEP_UV: u32 = 6_250; + +/// Safety envelope for the A76 rails. Floor = the lowest OPP nominal (675 mV); +/// ceiling = the top OPP voltage (1.0 V @ 2256 MHz). `set_uv*` refuse anything +/// outside. The governor only ever passes OPP-matched voltages (each a +/// Linux-proven freq/voltage pair), so any accepted value is proven-safe; the +/// clamp is defense-in-depth against a bad caller. (Phase-1a only sets 675 mV.) +const VDD_FLOOR_UV: u32 = 675_000; +const VDD_CEIL_UV: u32 = 1_000_000; + +/// Maximum voltage change per stepped-lowering step (≤ 4 LSB = 25 mV), so the +/// voltage-coupled clock tracks the rail down gradually. +const STEP_MAX_UV: u32 = 25_000; +/// Settle delay after each step, giving the rail time to slew before the next. +const SETTLE_US: u64 = 500; + +// --------------------------------------------------------------------------- +// Voltage <-> VSEL code (pure, host-testable) +// --------------------------------------------------------------------------- + +/// Decode an 8-bit VSEL code to microvolts. +fn vsel_to_uv(vsel: u8) -> u32 { + VSEL_BASE_UV + (vsel as u32) * VSEL_STEP_UV +} + +/// Encode microvolts to a VSEL code, requiring an **exact** round-trip. Returns +/// `None` for values below the base, above the 8-bit range, or that do not land +/// exactly on a 6.25 mV step (so we never program an approximated voltage). +fn uv_to_vsel(uv: u32) -> Option { + if uv < VSEL_BASE_UV { + return None; + } + let code = (uv - VSEL_BASE_UV) / VSEL_STEP_UV; + if code > VSEL_MASK as u32 { + return None; + } + let vsel = code as u8; + (vsel_to_uv(vsel) == uv).then_some(vsel) +} + +/// Whether `uv` is inside the Phase-2 down-only safety envelope. +fn in_envelope(uv: u32) -> bool { + (VDD_FLOOR_UV..=VDD_CEIL_UV).contains(&uv) +} + +// --------------------------------------------------------------------------- +// rk3x_i2c polling master +// --------------------------------------------------------------------------- + +/// Outcome of waiting on an I2C completion event. +enum I2cErr { + Nak, + Timeout, +} + +struct Rk3xI2c { + mmio: MmioRaw, +} + +impl Rk3xI2c { + #[inline] + fn r(&self, off: usize) -> u32 { + self.mmio.read::(off) + } + + #[inline] + fn w(&self, off: usize, val: u32) { + self.mmio.write::(off, val); + } + + /// Prepare the controller for polled use. u-boot has already clocked and + /// used this bus, so its `CLKDIV` is a proven-good SCL rate; preserve it and + /// only fall back to a conservative ~100 kHz divider (assuming RK3588's + /// 200 MHz i2c function clock) if it is somehow unset. Then park the + /// controller disabled with interrupts off and pending status cleared. + fn init_controller(&self) { + if self.r(REG_CLKDIV) == 0 { + // div = ceil(200 MHz / (100 kHz · 8)) - 2 = 248; split high/low. + let (divl, divh) = (124u32, 124u32); + self.w(REG_CLKDIV, (divh << 16) | divl); + warn!("pmic_i2c: CLKDIV was 0; set conservative ~100 kHz (divl=divh=124 @200 MHz)"); + } + self.w(REG_CON, 0); + self.w(REG_IEN, 0); + self.w(REG_IPD, INT_ALL); + info!( + "pmic_i2c: rk3x_i2c@{:#x} ready (clkdiv={:#x})", + RK3588_I2C0_BASE, + self.r(REG_CLKDIV) + ); + } + + /// Poll IPD until any bit in `mask` is set. A received NACK aborts early. + fn wait_ipd(&self, mask: u32) -> Result<(), I2cErr> { + for _ in 0..I2C_POLL_MAX { + let ipd = self.r(REG_IPD); + if ipd & INT_NAKRCV != 0 { + self.w(REG_IPD, INT_NAKRCV); + return Err(I2cErr::Nak); + } + if ipd & mask != 0 { + self.w(REG_IPD, mask); + return Ok(()); + } + axklib::time::busy_wait(Duration::from_micros(1)); + } + Err(I2cErr::Timeout) + } + + fn send_start(&self) -> Result<(), I2cErr> { + self.w(REG_IPD, INT_ALL); + self.w(REG_CON, CON_EN | CON_START); + self.w(REG_IEN, INT_START); + let res = self.wait_ipd(INT_START); + if res.is_err() { + // Board diagnostic: if START never completes, dump the controller + // state. A live controller sets INT_START(bit4) in IPD; CON reads + // back EN|START; CLKDIV should hold the (u-boot) divisor. All-zero + // or stuck values point at a still-gated clock or held reset. + warn!( + "pmic_i2c: START did not complete: CON={:#010x} IPD={:#010x} CLKDIV={:#010x} \ + IEN={:#010x}", + self.r(REG_CON), + self.r(REG_IPD), + self.r(REG_CLKDIV), + self.r(REG_IEN) + ); + } + res + } + + fn send_stop(&self) -> Result<(), I2cErr> { + self.w(REG_IPD, INT_ALL); + self.w(REG_CON, CON_EN | CON_STOP); + self.w(REG_IEN, INT_STOP); + self.wait_ipd(INT_STOP) + } + + #[inline] + fn disable(&self) { + self.w(REG_CON, 0); + } + + /// SMBus-style single-byte register write: `[START][chip.W][reg][val][STOP]`. + fn write_reg(&self, chip: u8, reg: u8, val: u8) -> bool { + if self.send_start().is_err() { + warn!("pmic_i2c: START failed writing chip {chip:#x} reg {reg:#x}"); + self.disable(); + return false; + } + // tx FIFO word 0: byte0 = write address, byte1 = reg, byte2 = data. + let word0 = ((chip as u32) << 1) | ((reg as u32) << 8) | ((val as u32) << 16); + self.w(TXDATA_BASE, word0); + self.w(REG_CON, CON_EN | con_mod(MODE_TX)); + self.w(REG_MTXCNT, 3); + self.w(REG_IEN, INT_MBTF | INT_NAKRCV); + let ok = self.wait_ipd(INT_MBTF); + // STOP before disable (disabling first would emit an illegal START/STOP). + let _ = self.send_stop(); + self.disable(); + match ok { + Ok(()) => true, + Err(I2cErr::Nak) => { + warn!("pmic_i2c: NACK writing chip {chip:#x} reg {reg:#x}"); + false + } + Err(I2cErr::Timeout) => { + warn!("pmic_i2c: timeout writing chip {chip:#x} reg {reg:#x}"); + false + } + } + } + + /// SMBus-style single-byte register read: combined write-address + read. + fn read_reg(&self, chip: u8, reg: u8) -> Option { + if self.send_start().is_err() { + warn!("pmic_i2c: START failed reading chip {chip:#x} reg {reg:#x}"); + self.disable(); + return None; + } + // Read address (byte0 valid) and the register byte to send (byte0 valid). + self.w(REG_MRXADDR, (((chip as u32) << 1) | 1) | MRXADDR_VALID0); + self.w(REG_MRXRADDR, (reg as u32) | MRXADDR_VALID0); + self.w(REG_CON, CON_EN | CON_LASTACK | con_mod(MODE_TRX)); + self.w(REG_MRXCNT, 1); + self.w(REG_IEN, INT_MBRF | INT_NAKRCV); + let res = self.wait_ipd(INT_MBRF); + let val = match res { + Ok(()) => Some((self.r(RXDATA_BASE) & 0xff) as u8), + Err(I2cErr::Nak) => { + warn!("pmic_i2c: NACK reading chip {chip:#x} reg {reg:#x}"); + None + } + Err(I2cErr::Timeout) => { + warn!("pmic_i2c: timeout reading chip {chip:#x} reg {reg:#x}"); + None + } + }; + let _ = self.send_stop(); + self.disable(); + val + } + + /// Write one VSEL code and confirm the read-back matches exactly. + fn set_vsel_verify(&self, chip: u8, vsel: u8) -> bool { + if !self.write_reg(chip, VSEL_REG, vsel) { + return false; + } + match self.read_reg(chip, VSEL_REG) { + Some(rb) if (rb & VSEL_MASK) == vsel => true, + Some(rb) => { + warn!( + "pmic_i2c: chip {chip:#x} VSEL read-back {:#x} != written {vsel:#x}", + rb & VSEL_MASK + ); + false + } + None => { + warn!("pmic_i2c: chip {chip:#x} VSEL read-back failed after write"); + false + } + } + } +} + +// --------------------------------------------------------------------------- +// Public API — single global PMIC bus (mapped once) +// --------------------------------------------------------------------------- + +static CONTROLLER: Mutex> = Mutex::new(None); + +/// Ungate the i2c0 PCLK + functional clock in the PMU CRU so the controller can +/// generate SCL. Idempotent — the Rockchip CRU gate registers are write-masked +/// (high 16 bits select which bits to write, low 16 are the values), and a `0` +/// in a gate bit means *enabled*, so re-clearing an already-enabled gate is a +/// no-op. Best-effort: on a mapping failure it logs and returns, leaving the +/// clock as-is (the subsequent transaction then times out into a safe no-op). +/// The controller is also released from soft-reset separately, see +/// [`deassert_i2c0_reset`]. +fn ungate_i2c0_clocks() { + let virt = match iomap(RK3588_PMU_CRU_BASE, RK3588_PMU_CRU_SIZE) { + Ok(ptr) => ptr, + Err(err) => { + warn!( + "pmic_i2c: iomap PMU CRU {RK3588_PMU_CRU_BASE:#x} failed: {err:?}; i2c0 clock \ + left as-is" + ); + return; + } + }; + // Clear both gate bits (write mask in the upper half, zeros in the lower). + let val = I2C0_GATE_BITS << 16; + // SAFETY: `virt` maps `RK3588_PMU_CRU_SIZE` bytes at the PMU CRU base; + // `PMU_CLKGATE_CON2` (0x808) is within that window. + let gate = unsafe { virt.as_ptr().add(PMU_CLKGATE_CON2).cast::() }; + unsafe { gate.write_volatile(val) }; + // Let the newly-ungated functional clock start before the first transfer. + axklib::time::busy_wait(Duration::from_micros(5)); + // Read back the (unmasked) gate bits for board diagnostics: 0 == enabled. + let after = unsafe { gate.read_volatile() } & I2C0_GATE_BITS; + info!( + "pmic_i2c: ungated i2c0 clocks (PMU_CLKGATE_CON(2)@{:#x} <- {val:#010x}); gate bits now \ + {after:#x} (0 == enabled)", + RK3588_PMU_CRU_BASE + PMU_CLKGATE_CON2 + ); +} + +/// Release the i2c0 controller from soft-reset in the PMU CRU so its SCL state +/// machine can run. De-assert only (write masked zeros to SRST_P_I2C0 bit 1 + +/// SRST_I2C0 bit 2); we never assert, so the PCLK-domain CLKDIV that u-boot left +/// is preserved. Idempotent (re-clearing an already-released reset is a no-op). +/// Best-effort: a mapping failure is logged and the later transaction just times +/// out into a safe no-op. Reads the bits back for board diagnostics. +fn deassert_i2c0_reset() { + let virt = match iomap(RK3588_PMU_CRU_BASE, RK3588_PMU_CRU_SIZE) { + Ok(ptr) => ptr, + Err(err) => { + warn!( + "pmic_i2c: iomap PMU CRU {RK3588_PMU_CRU_BASE:#x} failed: {err:?}; i2c0 reset \ + left as-is" + ); + return; + } + }; + // Clear both reset bits (write mask in the upper half, zeros in the lower). + let val = I2C0_RESET_BITS << 16; + // SAFETY: `virt` maps `RK3588_PMU_CRU_SIZE` bytes at the PMU CRU base; + // `PMU_SOFTRST_CON2` (0xa08) is within that window. + let softrst = unsafe { virt.as_ptr().add(PMU_SOFTRST_CON2).cast::() }; + unsafe { softrst.write_volatile(val) }; + // Give the core a moment to come out of reset before the first transfer. + axklib::time::busy_wait(Duration::from_micros(10)); + // Read back the (unmasked) reset bits for board diagnostics: 0 == released. + let after = unsafe { softrst.read_volatile() } & I2C0_RESET_BITS; + info!( + "pmic_i2c: de-asserted i2c0 reset (PMU_SOFTRST_CON(2)@{:#x} <- {val:#010x}); reset bits \ + now {after:#x} (0 == released)", + RK3588_PMU_CRU_BASE + PMU_SOFTRST_CON2 + ); +} + +/// Mux the i2c0 SCL/SDA pads to the i2c0 function via the registered Rockchip +/// pinctrl driver. Without this the rk3x master waits forever for a free bus and +/// never generates START (observed on-board: `IPD=0` after ungate + reset): +/// u-boot muxed these pads to i2c0 to program the PMICs, but StarryOS reverts +/// them at boot. The pins are i2c0m2 — `GPIO0_D1` (SCL) / `GPIO0_D2` (SDA) at +/// function 3 — which sit in the GPIO0_D "high group" split PMU2_IOC/BUS_IOC +/// mux path; rather than hand-decode that, we apply the i2c0 node's `pinctrl-0` +/// state through the in-tree pinctrl driver, which already encodes it. This is a +/// mux change only (no PMIC access). Best-effort: any failure is logged and the +/// later transaction times out into a safe no-op. +fn set_i2c0_pinmux() { + let Some(pinctrl) = rdrive::get_one::() else { + warn!("pmic_i2c: PinctrlDevice not registered; cannot mux i2c0 pins (START will fail)"); + return; + }; + let mut pinctrl = match pinctrl.lock() { + Ok(guard) => guard, + Err(err) => { + warn!("pmic_i2c: failed to lock PinctrlDevice: {err}; cannot mux i2c0 pins"); + return; + } + }; + let Some(fdt) = rdrive::with_fdt(Clone::clone) else { + warn!("pmic_i2c: live FDT not found; cannot mux i2c0 pins"); + return; + }; + // The i2c0 controller node (reg base == RK3588_I2C0_BASE) carries + // `pinctrl-0 = <&i2c0m2_xfer>`; several i2c controllers share the + // rk3399-i2c compatible, so match on the reg address. + let node = fdt + .find_compatible(&["rockchip,rk3588-i2c", "rockchip,rk3399-i2c"]) + .into_iter() + .find(|n| { + n.regs() + .into_iter() + .next() + .is_some_and(|r| r.address as usize == RK3588_I2C0_BASE) + }); + let Some(node) = node else { + warn!("pmic_i2c: i2c0 node @ {RK3588_I2C0_BASE:#x} not found in FDT; cannot mux pins"); + return; + }; + match pinctrl.apply_fdt_default_state(&fdt, node.as_node()) { + Ok(()) => info!( + "pmic_i2c: applied i2c0 pinctrl-0 (scl GPIO0_D1 / sda GPIO0_D2 -> func3) via rockchip \ + pinctrl" + ), + Err(err) => warn!("pmic_i2c: failed to apply i2c0 pinctrl-0: {err:?}"), + } +} + +/// Map and initialise the RK3588 PMIC I2C controller. Idempotent; safe to call +/// from a `PostKernel` probe context (MMU up). Returns `false` if the MMIO +/// mapping fails, in which case every `get_uv`/`set_uv*` below is a no-op that +/// returns `None`/`false` and leaves the rails at their (safe) boot voltage. +pub fn init() -> bool { + let mut guard = CONTROLLER.lock(); + if guard.is_some() { + return true; + } + // Bring the i2c0 bus up before any transaction: ungate its clocks, release + // it from soft-reset, and mux its SCL/SDA pads to the i2c0 function (all + // three left closed/reverted at boot; any one makes every START time out). + // All best-effort — a failure just leaves the later transaction to time out + // into a safe no-op. + ungate_i2c0_clocks(); + deassert_i2c0_reset(); + set_i2c0_pinmux(); + let virt = match iomap(RK3588_I2C0_BASE, RK3588_I2C0_SIZE) { + Ok(ptr) => ptr, + Err(err) => { + warn!("pmic_i2c: iomap {RK3588_I2C0_BASE:#x} failed: {err:?}"); + return false; + } + }; + // SAFETY: `virt` is a fresh device mapping of `RK3588_I2C0_BASE` of + // `RK3588_I2C0_SIZE` bytes returned by `iomap`. + let mmio = unsafe { + MmioRaw::new( + MmioAddr::from(RK3588_I2C0_BASE as u64), + virt, + RK3588_I2C0_SIZE, + ) + }; + let i2c = Rk3xI2c { mmio }; + i2c.init_controller(); + *guard = Some(i2c); + true +} + +/// Read a rail's current voltage in microvolts. `chip` is [`RK8602_BIG0_ADDR`] +/// or [`RK8603_BIG1_ADDR`]. `None` if the bus is uninitialised or the read +/// fails. +pub fn get_uv(chip: u8) -> Option { + let guard = CONTROLLER.lock(); + let i2c = guard.as_ref()?; + let vsel = i2c.read_reg(chip, VSEL_REG)?; + Some(vsel_to_uv(vsel & VSEL_MASK)) +} + +/// Set a rail directly to `target_uv` (single write), refusing anything outside +/// the Phase-2 envelope `[675_000, 800_000]` µV or that is not an exact 6.25 mV +/// step, and verifying the read-back. Returns `false` on any rejection or +/// mismatch (leaving the rail unchanged/at its last confirmed value). +/// +/// This is the direct setter. Live-core lowering should use +/// [`set_uv_stepped`], which ramps down in small increments. +pub fn set_uv(chip: u8, target_uv: u32) -> bool { + if !in_envelope(target_uv) { + warn!( + "pmic_i2c: refusing chip {chip:#x} set to {target_uv} uV (outside [{VDD_FLOOR_UV}, \ + {VDD_CEIL_UV}])" + ); + return false; + } + let Some(vsel) = uv_to_vsel(target_uv) else { + warn!("pmic_i2c: {target_uv} uV is not an exact VSEL step; refusing"); + return false; + }; + let guard = CONTROLLER.lock(); + let Some(i2c) = guard.as_ref() else { + warn!("pmic_i2c: not initialised; refusing set"); + return false; + }; + i2c.set_vsel_verify(chip, vsel) +} + +/// Safely lower a rail to `target_uv` by stepping **down** in ≤25 mV (4-LSB) +/// increments, verifying the read-back and settling after each step, so the +/// voltage-coupled CPU clock tracks the rail down without a large undervolt +/// transient. This is the path `cpufreq` calls on live cores. +/// +/// Refuses (returns `false`) if `target_uv` is outside the envelope, is not an +/// exact step, or is **above** the rail's current voltage (this path never +/// raises voltage). A no-op success if already at target. Aborts and returns +/// `false` on the first read-back mismatch, leaving the rail at the last +/// verified step. +pub fn set_uv_stepped(chip: u8, target_uv: u32) -> bool { + if !in_envelope(target_uv) { + warn!( + "pmic_i2c: refusing chip {chip:#x} step to {target_uv} uV (outside [{VDD_FLOOR_UV}, \ + {VDD_CEIL_UV}])" + ); + return false; + } + let Some(target_vsel) = uv_to_vsel(target_uv) else { + warn!("pmic_i2c: {target_uv} uV is not an exact VSEL step; refusing"); + return false; + }; + + let guard = CONTROLLER.lock(); + let Some(i2c) = guard.as_ref() else { + warn!("pmic_i2c: not initialised; refusing step"); + return false; + }; + + let Some(cur_vsel) = i2c.read_reg(chip, VSEL_REG).map(|v| v & VSEL_MASK) else { + warn!("pmic_i2c: chip {chip:#x} current VSEL read failed; refusing step"); + return false; + }; + + if target_vsel > cur_vsel { + warn!( + "pmic_i2c: refusing to step UP chip {chip:#x} {} -> {target_uv} uV (down-only path)", + vsel_to_uv(cur_vsel) + ); + return false; + } + if target_vsel == cur_vsel { + return true; + } + + const STEP_LSB: u8 = (STEP_MAX_UV / VSEL_STEP_UV) as u8; // 25000 / 6250 = 4 + let mut v = cur_vsel; + while v > target_vsel { + let next = v.saturating_sub(STEP_LSB).max(target_vsel); + if !i2c.set_vsel_verify(chip, next) { + warn!( + "pmic_i2c: chip {chip:#x} step to VSEL {next:#x} ({} uV) failed; aborting at {} uV", + vsel_to_uv(next), + vsel_to_uv(v) + ); + return false; + } + axklib::time::busy_wait(Duration::from_micros(SETTLE_US)); + v = next; + } + info!( + "pmic_i2c: chip {chip:#x} stepped {} -> {} uV", + vsel_to_uv(cur_vsel), + vsel_to_uv(target_vsel) + ); + true +} + +#[cfg(test)] +mod tests { + use super::*; + + // Board-confirmed VSEL points. + #[test] + fn vsel_decode_matches_board_points() { + assert_eq!(vsel_to_uv(0x1c), 675_000); + assert_eq!(vsel_to_uv(0x20), 700_000); + assert_eq!(vsel_to_uv(0x24), 725_000); + assert_eq!(vsel_to_uv(0x30), 800_000); + assert_eq!(vsel_to_uv(0x50), 1_000_000); + } + + #[test] + fn uv_encode_roundtrips_board_points() { + assert_eq!(uv_to_vsel(675_000), Some(0x1c)); + assert_eq!(uv_to_vsel(700_000), Some(0x20)); + assert_eq!(uv_to_vsel(725_000), Some(0x24)); + assert_eq!(uv_to_vsel(800_000), Some(0x30)); + assert_eq!(uv_to_vsel(1_000_000), Some(0x50)); + } + + #[test] + fn uv_encode_rejects_non_step_and_out_of_range() { + // Not a multiple of 6.25 mV. + assert_eq!(uv_to_vsel(676_000), None); + assert_eq!(uv_to_vsel(675_001), None); + // Below the encoding base. + assert_eq!(uv_to_vsel(499_999), None); + // Above the 8-bit code range (code 255 -> 2.09375 V). + assert_eq!(uv_to_vsel(2_100_000), None); + } + + #[test] + fn envelope_is_down_only_675_to_800() { + assert!(in_envelope(675_000)); + assert!(in_envelope(800_000)); + assert!(in_envelope(725_000)); + assert!(!in_envelope(674_999)); // below OPP floor + assert!(!in_envelope(800_001)); // above boot voltage + assert!(!in_envelope(1_000_000)); + } + + #[test] + fn step_size_is_four_lsb_25mv() { + const STEP_LSB: u8 = (STEP_MAX_UV / VSEL_STEP_UV) as u8; + assert_eq!(STEP_LSB, 4); + assert_eq!((STEP_LSB as u32) * VSEL_STEP_UV, 25_000); + // 800 -> 675 mV is 20 LSB = exactly 5 whole 4-LSB steps. + assert_eq!( + uv_to_vsel(800_000).unwrap() - uv_to_vsel(675_000).unwrap(), + 20 + ); + } +} diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs b/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs new file mode 100644 index 0000000000..5db319e2ad --- /dev/null +++ b/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs @@ -0,0 +1,1044 @@ +// Copyright 2025 The Axvisor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RK3588 A55 (`vdd_cpu_lit`) CPU-rail voltage lever — RK806 PMIC over SPI2. +//! +//! DVFS Phase 2, little cluster. Phase 1a raises the A55 clock via SCMI only; +//! the RK3588 CPU clock is voltage-coupled, so exact frequency + power needs the +//! per-OPP rail voltage that only the PMIC can set. On the OrangePi-5-Plus the +//! A55 rail `vdd_cpu_lit_s0` is provided by an **RK806** PMIC on **SPI2** +//! (`spi@feb20000`, chip-select 0). This module is a minimal polling SPI master +//! plus the RK806 buck-2 regulator access needed to read and (safely, down-only) +//! lower that rail. +//! +//! # Board-confirmed ground truth (sources cited inline) +//! +//! - **SPI2 controller** = `spi@feb20000`, size `0x1000`, compatible +//! `rockchip,rk3066-spi` (the standard Rockchip SPI IP). Register map and the +//! bare-metal polling flow follow mainline `drivers/spi/spi-rockchip.c` and +//! U-Boot `drivers/spi/rk_spi.c` (`rk_spi.h`). +//! - **RK806** = `rockchip,rk806`, CS 0, `spi-max-frequency = 1_000_000`, mode 0. +//! SPI framing from mainline `drivers/mfd/rk8xx-spi.c` +//! (`rk806_spi_bus_read`/`rk806_spi_bus_write`) + constants in +//! `include/linux/mfd/rk808.h`. +//! - **`vdd_cpu_lit_s0` = RK806 `dcdc-reg2` (buck #2)** — from +//! `arch/arm64/boot/dts/rockchip/rk3588-orangepi-5.dtsi`. Its running voltage +//! selector is `RK806_BUCK2_ON_VSEL = 0x1B`, full-byte selector +//! (`vsel_mask = 0xff`), encoded by `rk806_buck_voltage_ranges` in +//! `drivers/regulator/rk808-regulator.c`: +//! `V(uV) = 500000 + sel*6250` for `sel 0..=159` (500 mV..1500 mV, our whole +//! CPU window). 675 mV = sel 28 (`0x1C`), 750 mV = sel 40 (`0x28`), +//! 800 mV = sel 48 (`0x30`). +//! +//! # Safety model +//! +//! Direction is **down only**, toward Linux-proven OPP nominals. [`set_uv`] +//! refuses any target below the [`A55_MIN_UV`] floor (675 mV) or **above the +//! boot voltage read back from the rail** — it can never raise vdd. Every write +//! is read-back verified (raw selector byte compared), and a mismatch aborts and +//! returns `false`, leaving the last confirmed selector in place. +//! [`set_uv_stepped`] lowers in `<=`[`MAX_STEP_UV`] (25 mV) increments with a +//! settle delay so the voltage-coupled clock tracks down gradually. +//! +//! After U-Boot, SPI2 is left with its clocks gated **and** its controller held +//! in soft-reset, so [`init`] first **ungates `PCLK_SPI2` + `CLK_SPI2`**, +//! **de-asserts the SPI2 P+core soft-reset**, and **applies the full SPI2 +//! `pinctrl-0`** (mux + pull + drive + schmitt, via the in-tree rockchip pinctrl +//! driver — a mux-only match is not enough; the pad config is what the RK806 link +//! needs, mirroring the i2c0 fix), then verifies the controller responds (a gated +//! APB reads all-zero, and a core still in reset transacts all-zero — either +//! would otherwise clock in a fabricated `0`); if it does not respond it stays +//! **unbound**. It also logs a one-shot read-only RK806 reachability probe +//! (chip-ID + raw DCDC2 frame). Every SPI poll loop is time-bounded +//! ([`SPI_TIMEOUT_NS`]) and treats an all-zero status register as a dead bus, so +//! a controller that dies after init makes [`get_uv`] return `None` rather than +//! hanging or fabricating a reading. `init`'s only SoC writes are to the SPI2 +//! clock gate, soft-reset, and pin config; **no PMIC write** in `init`. +//! +//! # Integration (owned by the cpufreq probe, not wired here) +//! +//! This module performs **no** PMIC write on its own. Mirroring the A76 I2C +//! sibling, [`init`] maps the controller itself; the probe then reads/steps the +//! rail: +//! +//! ```ignore +//! use crate::soc::rockchip::pmic_spi; +//! +//! pmic_spi::init(); // map + configure SPI2/RK806 +//! let boot_uv = pmic_spi::get_uv(); // read A55 boot vdd first +//! // ... only after the SCMI clock is already at target: +//! pmic_spi::set_uv_stepped(675_000); // down-only to the OPP nominal +//! ``` + +use core::time::Duration; + +use ax_kspin::SpinNoIrq as Mutex; +use log::{info, warn}; +use rdif_pinctrl::PinctrlDevice; + +use crate::mmio::iomap; + +// --------------------------------------------------------------------------- +// SPI2 controller (rockchip,rk3066-spi) — spi@feb20000 +// --------------------------------------------------------------------------- + +/// Physical base of the RK3588 SPI2 controller (`spi@feb20000`). +pub const RK3588_SPI2_BASE: usize = 0xfeb2_0000; +/// MMIO window size of the SPI2 controller. +pub const RK3588_SPI2_SIZE: usize = 0x1000; +/// Fallback SPI2 functional-clock rate for the baud divider, used only if +/// U-Boot left no usable divisor (see [`Rk806Spi::configure`]). +/// +/// The OrangePi-5-Plus DT sets `assigned-clock-rates = <200000000>` for +/// `CLK_SPI2`; 200 MHz is also the RK3588 SPI source ceiling, so assuming it can +/// only *under*-shoot the target SCLK (safe: slower, still <= 1 MHz). +const RK3588_SPI2_FALLBACK_INPUT_HZ: u32 = 200_000_000; + +/// Target serial clock for the RK806 (DT `spi-max-frequency = 1_000_000`). +const SPI_SCLK_HZ: u32 = 1_000_000; +/// RK806 chip-select index on SPI2. +const RK806_CS: u32 = 0; + +// Register offsets (spi-rockchip.c ROCKCHIP_SPI_* / rk_spi.h struct layout). +const SPI_CTRLR0: usize = 0x0000; +const SPI_CTRLR1: usize = 0x0004; +const SPI_ENR: usize = 0x0008; +const SPI_SER: usize = 0x000c; +const SPI_BAUDR: usize = 0x0010; +const SPI_RXFTLR: usize = 0x0018; +const SPI_SR: usize = 0x0024; +const SPI_IMR: usize = 0x002c; +const SPI_ICR: usize = 0x0038; +const SPI_TXDR: usize = 0x0400; +const SPI_RXDR: usize = 0x0800; + +// CTRLR0 fields (rk_spi.h). We now use the EXACT value Linux `spi-rockchip.c` +// programs on this board (read live: 0x2C01), since Linux reads the RK806 with +// it — master, 8-bit frame, mode 0, transmit+receive, EM_BIG. +const CR0_DFS_8BIT: u32 = 0x1; // [1:0] data frame size = 8 bit +const CR0_SSN_DELAY_ONE: u32 = 0x1 << 10; // one sclk CS-to-clk delay +const CR0_EM_BIG: u32 = 0x1 << 11; // endian: big (matches Linux spi-rockchip.c) +const CR0_HALF_WORD_OFF: u32 = 0x1 << 13; // APB 8-bit / SPI 8-bit access +// FRF_SPI (0<<16), TMOD_TR (0<<18), OMOD_MASTER (0<<20), FBM_MSB (0<<12), +// SCPH/SCPOL (mode 0 => 0<<6/0<<7), RXD sample delay 0 all contribute zero. +const SPI_CTRLR0_MODE0_8BIT: u32 = + CR0_DFS_8BIT | CR0_SSN_DELAY_ONE | CR0_EM_BIG | CR0_HALF_WORD_OFF; + +// Status register bits (rk_spi.h). +const SR_BUSY: u32 = 1 << 0; +const SR_TF_FULL: u32 = 1 << 1; +const SR_RF_EMPT: u32 = 1 << 3; + +const BAUDR_MIN: u32 = 2; +const BAUDR_MAX: u32 = 0xfffe; + +// Main RK3588 CRU (write-masked clock/reset registers). U-Boot gates the SPI2 +// clocks after using the RK806 at boot, so `init` must ungate them first — a +// gated APB reads all-zero, which the controller-alive check below rejects. +/// Main RK3588 CRU base. +const RK3588_CRU_BASE: usize = 0xfd7c_0000; +/// CRU window mapped for the ungate (only the first page is needed). +const RK3588_CRU_SIZE: usize = 0x1000; +/// `clkgate_con(14)` (`14*4 + 0x800`): holds the SPI2 gate bits. +const CRU_CLKGATE_CON14: usize = 0x838; +/// Write-masked ungate of `PCLK_SPI2` (bit 8) + `CLK_SPI2` (bit 13): high half +/// is the write-enable mask, low half is the value (0 == clock enabled on a +/// Rockchip CRU gate). == `0x2100_0000`. +const SPI2_CLK_UNGATE: u32 = ((1 << 8) | (1 << 13)) << 16; + +/// Upper bound on any SPI poll loop. Generous vs. the ~40 us a 4-byte 1 MHz +/// frame takes; a dead SPI clock trips this instead of hanging. +const SPI_TIMEOUT_NS: u64 = 20_000_000; // 20 ms + +/// Largest RK806 transaction this module issues: cmd + 2 addr + 1 data. +const RK806_FRAME_LEN: usize = 4; + +// --------------------------------------------------------------------------- +// RK806 SPI framing (rk8xx-spi.c / rk808.h) +// --------------------------------------------------------------------------- + +/// Command byte: write. `RK806_CMD_WRITE = BIT(7)`; CRC disabled (`= 0`); the +/// low nibble carries `value_bytes - 1`, which is 0 for our single-byte access. +const RK806_CMD_WRITE: u8 = 0x80; +/// Command byte: read. `RK806_CMD_READ = 0`; CRC disabled; low nibble 0. +const RK806_CMD_READ: u8 = 0x00; + +/// RK806 buck-2 (`vdd_cpu_lit_s0`) running-voltage selector register. +const RK806_BUCK2_ON_VSEL: u8 = 0x1B; + +/// RK806 chip-name / chip-version registers (rk808.h `RK806_CHIP_NAME`/`_VER`). +/// Both carry nonzero power-on defaults, so a nonzero read proves the SPI path +/// actually reaches the RK806 (vs. reading fabricated zeros off a floating MISO). +const RK806_CHIP_NAME: u8 = 0x5A; +const RK806_CHIP_VER: u8 = 0x5B; + +// --- SPI2 controller soft-reset (RK3588 main CRU) ------------------------- +// Board-observed (boot #2): after ungate the controller is clocked and its APB +// registers stick (`is_alive` passes), yet the serial transaction reads all-zero +// — the classic signature of the controller **core** held in soft-reset: the +// PCLK-domain reset is released (registers work) but the shift-engine reset is +// held, so nothing actually transacts. Same split the i2c0 sibling hit. SPI2's +// resets live in the MAIN CRU: SRST_P_SPI2 (PCLK) = bit 8 and SRST_SPI2 (core) = +// bit 13 of `softrst_con(14)`, from mainline `drivers/clk/rockchip/rst-rk3588.c` +// (`RK3588_CRU_RESET_OFFSET(SRST_P_SPI2,14,8)` / `(SRST_SPI2,14,13)`) — same bit +// layout as the clock gate. We de-assert both (0 == released); the core reset +// (bit 13) is the one that matches the symptom. +/// `softrst_con(14)` (`14*4 + 0xa00`): holds the SPI2 reset bits. +const CRU_SOFTRST_CON14: usize = 0xa38; +/// Write-masked de-assert of SRST_P_SPI2 (bit 8) + SRST_SPI2 (bit 13): high half +/// = write mask, low half = 0 (0 == reset released). Numerically equals the gate +/// ungate value (same bit positions) but targets a different register. +const SPI2_RST_DEASSERT: u32 = ((1 << 8) | (1 << 13)) << 16; +/// Settle after releasing the reset before touching the controller. +const RST_SETTLE_US: u64 = 20; + +// RK3588 PMU1_IOC (@ 0xfd5f0000) GPIO0 A/B pad config, decoded from u-boot +// pinctrl-rk3588.c tables (verified against Linux's live values): +// mux: 0x04 = GPIO0A_H (A4-7), 0x08 = GPIO0B_L (B0-3) [4 bits/pin] +// drive: 0x14 = GPIO0A_H, 0x18 = GPIO0B_L [4 bits/pin] +// pull: 0x20 = GPIO0A, 0x24 = GPIO0B [2 bits/pin] +// smt: 0x30 = GPIO0A, 0x34 = GPIO0B [1 bit/pin] +// SPI2 pins: A5=CLK, A6=MOSI (in 0x04/0x14/0x20/0x30), B1=CS0, B3=MISO (in +// 0x08/0x18/0x24/0x34). The read-only dump below logs this whole region so it +// can be diffed against Linux's known-good reference to find the B3/MISO delta. +const RK3588_IOC_BASE: usize = 0xfd5f_0000; +const RK3588_IOC_SIZE: usize = 0x1000; + +// --------------------------------------------------------------------------- +// RK806 buck voltage encoding (rk806_buck_voltage_ranges, range 1) +// --------------------------------------------------------------------------- + +/// Range-1 base: selector 0 == 500 mV. +const RK806_VSEL_BASE_UV: u32 = 500_000; +/// Range-1 step: 6.25 mV per selector unit. +const RK806_VSEL_STEP_UV: u32 = 6_250; +/// Last selector of range 1 (== 1_493_750 uV; selector 160 is the range-2 base +/// at 1_500_000 uV). Linux's "500mV ~ 1500mV" range comment is approximate. +const RK806_VSEL_R1_MAX_SEL: u8 = 159; +/// Range-2 base (selector 160 == 1_500_000 uV), 25 mV step, up to selector 235. +const RK806_VSEL_R2_BASE_UV: u32 = 1_500_000; +const RK806_VSEL_R2_BASE_SEL: u8 = 160; +const RK806_VSEL_R2_STEP_UV: u32 = 25_000; +const RK806_VSEL_R2_MAX_SEL: u8 = 235; +/// Range-3: selector 236..=255 is a fixed 3_400_000 uV. +const RK806_VSEL_R3_UV: u32 = 3_400_000; + +// --------------------------------------------------------------------------- +// Safety envelope for the A55 rail +// --------------------------------------------------------------------------- + +/// Hard voltage floor for the A55 rail: the lowest Phase-1a OPP nominal +/// (1008 MHz -> 675 mV, standard SKU). Never set below this. Industrial J/M SKU +/// low OPPs sit at 750 mV; a boot read of ~750 is expected there and is fine. +pub const A55_MIN_UV: u32 = 675_000; +/// Maximum single-step magnitude for [`set_uv_stepped`] (25 mV == 4 selectors). +pub const MAX_STEP_UV: u32 = 25_000; +/// Settle delay between stepped-lowering writes (>> the 25 mV ramp time at the +/// DT `regulator-ramp-delay = 12500` uV/us => ~2 us). +const STEP_SETTLE_US: u64 = 150; + +// --------------------------------------------------------------------------- +// Controller handle + global state +// --------------------------------------------------------------------------- + +/// A mapped, initialized SPI2 controller bound to the RK806. +struct Rk806Spi { + base: *mut u8, +} + +// The controller is reached only through `PMIC` (a `SpinNoIrq` mutex), which +// serializes all access; the MMIO mapping is stable for the kernel lifetime. +unsafe impl Send for Rk806Spi {} + +/// Global RK806/SPI2 handle, populated once by [`init`]. +static PMIC: Mutex> = Mutex::new(None); + +impl Rk806Spi { + #[inline] + fn read(&self, off: usize) -> u32 { + unsafe { self.base.add(off).cast::().read_volatile() } + } + + #[inline] + fn write(&self, off: usize, val: u32) { + unsafe { self.base.add(off).cast::().write_volatile(val) } + } + + /// One-time controller setup: master, mode 0, 8-bit frames, ~1 MHz SCLK. + /// Idempotent; leaves the controller disabled (transfers enable per frame). + /// + /// U-Boot already drives the RK806 over SPI2 at boot, so its BAUDR is a + /// proven-good ~1 MHz divisor — preserve it (like the I2C sibling preserves + /// `CLKDIV`) and only compute one if the controller was left unconfigured. + /// This removes any dependence on knowing the exact `CLK_SPI2` rate. + fn configure(&self) { + // Disable before touching CTRLR0/BAUDR (config is latched while idle). + self.write(SPI_ENR, 0); + self.write(SPI_SER, 0); + + let baud = self.read(SPI_BAUDR) & 0xffff; + if (BAUDR_MIN..=BAUDR_MAX).contains(&baud) && (baud & 1) == 0 { + info!("pmic_spi: SPI2 baud divisor preserved from U-Boot ({baud})"); + } else { + let computed = baud_divider(RK3588_SPI2_FALLBACK_INPUT_HZ, SPI_SCLK_HZ); + self.write(SPI_BAUDR, computed); + info!("pmic_spi: SPI2 baud divisor computed {computed} (U-Boot left {baud:#x})"); + } + + self.write(SPI_CTRLR0, SPI_CTRLR0_MODE0_8BIT); + // Mask + clear any interrupts; this is a poll-only master. + self.write(SPI_IMR, 0); + self.write(SPI_ICR, 0xffff_ffff); + } + + /// Confirm the controller actually responds on its APB. `configure` always + /// writes the fixed `SPI_CTRLR0_MODE0_8BIT`, so those bits reading back + /// proves the bus is clocked and out of reset. A still-gated (or reset-held) + /// controller reads 0 here, which fails the mask check — this is what closes + /// the "gated APB looks like an empty-but-working bus" false-success hole. + fn is_alive(&self) -> bool { + self.read(SPI_CTRLR0) & SPI_CTRLR0_MODE0_8BIT == SPI_CTRLR0_MODE0_8BIT + } + + /// Full-duplex byte exchange of `tx.len()` bytes, CS asserted for the whole + /// transaction. Replicates Linux `spi-rockchip.c`'s exact PIO sequence (the + /// board's known-good driver uses an identical CTRLR0 to ours yet reads the + /// RK806, so the transfer *sequence* is the remaining variable): assert CS, + /// program CTRLR0/CTRLR1/**RXFTLR** while disabled, clear interrupts, enable, + /// **pre-fill the entire TX FIFO**, then drain RX. `rx[3]` holds the value. + /// Returns `false` on timeout, leaving CS deasserted and the controller off. + #[must_use] + fn xfer(&self, tx: &[u8], rx: &mut [u8]) -> bool { + self.xfer_cs(tx, rx, true) + } + + /// As [`Rk806Spi::xfer`], but `native_cs` selects whether the controller's + /// own CS (SER) is asserted. The GPIO-CS diagnostic drives the CS pin + /// manually and calls this with `native_cs = false`. + #[must_use] + fn xfer_cs(&self, tx: &[u8], rx: &mut [u8], native_cs: bool) -> bool { + debug_assert_eq!(tx.len(), rx.len()); + let len = tx.len(); + let n = len as u32; + + // set_cs first (Linux calls set_cs before the transfer). + if native_cs { + self.write(SPI_SER, 1 << RK806_CS); + } + + // Per-transfer config while disabled: CTRLR0, frame count, and the RX + // FIFO threshold (Linux sets RXFTLR = len-1 for small transfers; we + // previously left it unset — a prime suspect). + self.write(SPI_ENR, 0); + self.write(SPI_CTRLR0, SPI_CTRLR0_MODE0_8BIT); + self.write(SPI_CTRLR1, n.saturating_sub(1)); + self.write(SPI_RXFTLR, n.saturating_sub(1)); + self.write(SPI_ICR, 0xffff_ffff); + self.write(SPI_ENR, 1); + + let deadline = now_ns() + SPI_TIMEOUT_NS; + let mut ok = true; + + // Pre-fill the ENTIRE TX FIFO first (Linux fills then waits), rather than + // interleaving TX/RX. The 4-byte frame fits the FIFO with room to spare. + let mut ti = 0usize; + while ti < len { + let sr = self.read(SPI_SR); + if sr == 0 { + warn!("pmic_spi: SPI2 SR reads 0 (controller not clocking); aborting"); + ok = false; + break; + } + if (sr & SR_TF_FULL) == 0 { + self.write(SPI_TXDR, u32::from(tx[ti])); + ti += 1; + } else if now_ns() >= deadline { + warn!("pmic_spi: SPI2 TX fill timed out ({ti}/{len})"); + ok = false; + break; + } + } + + // Then drain the RX FIFO (one byte per clocked frame). + let mut ri = 0usize; + while ok && ri < len { + if (self.read(SPI_SR) & SR_RF_EMPT) == 0 { + rx[ri] = self.read(SPI_RXDR) as u8; + ri += 1; + } else if now_ns() >= deadline { + warn!("pmic_spi: SPI2 RX drain timed out ({ri}/{len})"); + ok = false; + break; + } + } + + // Wait for the shift engine to go idle before dropping CS. + while ok && (self.read(SPI_SR) & SR_BUSY) != 0 { + if now_ns() >= deadline { + warn!("pmic_spi: SPI2 stayed BUSY after transfer"); + ok = false; + break; + } + } + + // Deassert CS0 and disable. + if native_cs { + self.write(SPI_SER, 0); + } + self.write(SPI_ENR, 0); + ok + } + + /// Read one RK806 register (8-bit value, 16-bit little-endian address). + /// Frame: `[READ, addr_lo, addr_hi=0, dummy]`; value is the last RX byte. + fn rk806_read(&self, reg: u8) -> Option { + let tx = [RK806_CMD_READ, reg, 0x00, 0x00]; + let mut rx = [0u8; RK806_FRAME_LEN]; + if self.xfer(&tx, &mut rx) { + Some(rx[RK806_FRAME_LEN - 1]) + } else { + None + } + } + + /// Write one RK806 register. Frame: `[WRITE, addr_lo, addr_hi=0, value]`. + /// RX is ignored. Returns `false` on transfer timeout. + #[must_use] + fn rk806_write(&self, reg: u8, val: u8) -> bool { + let tx = [RK806_CMD_WRITE, reg, 0x00, val]; + let mut rx = [0u8; RK806_FRAME_LEN]; + self.xfer(&tx, &mut rx) + } + + /// Read-only one-shot reachability probe, logged at init. The RK806 + /// chip-name/version registers have nonzero power-on defaults, so a nonzero + /// read proves the SPI path actually reaches the RK806; an all-zero read (and + /// an all-zero DCDC2 frame) means it is not reached (pin-mux / CS). Also dumps + /// the raw DCDC2 read frame for inspection. Performs no PMIC write. + fn log_diagnostics(&self) { + let name = self.rk806_read(RK806_CHIP_NAME); + let ver = self.rk806_read(RK806_CHIP_VER); + let tx = [RK806_CMD_READ, RK806_BUCK2_ON_VSEL, 0x00, 0x00]; + let mut rx = [0u8; RK806_FRAME_LEN]; + let ok = self.xfer(&tx, &mut rx); + info!( + "pmic_spi: RK806 probe: chip_name(0x5A)={name:?} chip_ver(0x5B)={ver:?} \ + dcdc2_raw_rx={rx:02x?} (xfer_ok={ok})" + ); + } + + /// Read-only read-shape diagnostic (boot #8). The RK806 read now returns data + /// but the RX mirrors the TX (e.g. chip-name read `tx=[00,5a,00,00]` came back + /// `rx=[00,5a,00,00]`). Distinguish a MISO-reads-MOSI loopback from a + /// value-position offset: + /// - **distinct**: send a distinctive TX pattern; if the RX echoes it byte for + /// byte, MISO is reading MOSI (pad/routing), not the RK806. + /// - **long6**: a 6-byte frame; if instead the register value lands at a later + /// wire byte (rx\[4]/rx\[5]), it is just a frame-length/index offset. + /// + /// No PMIC write. + fn probe_read_shape(&self) { + let tx1 = [RK806_CMD_READ, RK806_CHIP_NAME, 0xA5, 0x3C]; + let mut rx1 = [0u8; 4]; + let ok1 = self.xfer(&tx1, &mut rx1); + info!("pmic_spi: read-shape distinct: tx={tx1:02x?} rx={rx1:02x?} ok={ok1}"); + + let tx2 = [RK806_CMD_READ, RK806_CHIP_NAME, 0x00, 0x00, 0x00, 0x00]; + let mut rx2 = [0u8; 6]; + let ok2 = self.xfer(&tx2, &mut rx2); + info!("pmic_spi: read-shape long6: tx={tx2:02x?} rx={rx2:02x?} ok={ok2}"); + } +} + +// --------------------------------------------------------------------------- +// Baud + voltage math (pure, unit-tested on the host) +// --------------------------------------------------------------------------- + +/// SPI baud divider: even, in `[2, 0xfffe]`, rounded so the resulting SCLK is +/// `<= target_hz` (mirrors U-Boot `rkspi_set_clk`: `DIV_ROUND_UP` then round up +/// to the next even value). +fn baud_divider(input_hz: u32, target_hz: u32) -> u32 { + let target = target_hz.max(1); + let mut div = input_hz.div_ceil(target); + div = div.clamp(BAUDR_MIN, BAUDR_MAX); + // Round up to the next even number (hardware requires an even divisor); + // rounding up only lowers the clock, which is the safe direction. + div = (div + 1) & 0xfffe; + div.clamp(BAUDR_MIN, BAUDR_MAX) +} + +/// Decode an RK806 buck selector byte to microvolts (all three ranges). +fn vsel_to_uv(sel: u8) -> u32 { + if sel <= RK806_VSEL_R1_MAX_SEL { + RK806_VSEL_BASE_UV + u32::from(sel) * RK806_VSEL_STEP_UV + } else if sel <= RK806_VSEL_R2_MAX_SEL { + RK806_VSEL_R2_BASE_UV + u32::from(sel - RK806_VSEL_R2_BASE_SEL) * RK806_VSEL_R2_STEP_UV + } else { + RK806_VSEL_R3_UV + } +} + +/// Encode microvolts to an RK806 buck selector byte, range 1 only (500 mV.. +/// 1493.75 mV — spanning the entire CPU-rail window). Requires an **exact** +/// 6.25 mV-aligned value (like `pmic_i2c`'s encoder): returns `None` outside +/// range 1 or for any input that does not land exactly on a selector step, so +/// we never program an approximated voltage. Our OPP nominals (675/750/800 mV) +/// are all aligned. +fn uv_to_vsel(uv: u32) -> Option { + let r1_max_uv = RK806_VSEL_BASE_UV + u32::from(RK806_VSEL_R1_MAX_SEL) * RK806_VSEL_STEP_UV; + if uv < RK806_VSEL_BASE_UV || uv > r1_max_uv { + return None; + } + let offset = uv - RK806_VSEL_BASE_UV; + if !offset.is_multiple_of(RK806_VSEL_STEP_UV) { + return None; + } + Some((offset / RK806_VSEL_STEP_UV) as u8) +} + +// --------------------------------------------------------------------------- +// Time helpers +// --------------------------------------------------------------------------- + +#[inline] +fn now_ns() -> u64 { + axklib::time::monotonic_nanos() +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/// Ungate the SPI2 controller clocks (`PCLK_SPI2` + `CLK_SPI2`) in the main CRU. +/// +/// Board-observed: U-Boot gates these after programming the RK806 at boot, so at +/// probe time the SPI2 APB is dead and reads all-zero — which without this looks +/// like a working-but-empty bus and yields a bogus `0` voltage. The Rockchip CRU +/// gate register is write-masked, so writing [`SPI2_CLK_UNGATE`] clears both gate +/// bits (0 == enabled). Returns `false` only if the CRU cannot be mapped. +fn ungate_spi2_clocks() -> bool { + let cru = match iomap(RK3588_CRU_BASE, RK3588_CRU_SIZE) { + Ok(v) => v, + Err(err) => { + warn!("pmic_spi: iomap CRU {RK3588_CRU_BASE:#x} failed: {err:?}"); + return false; + } + }; + // SAFETY: the CRU mapping covers `RK3588_CRU_SIZE` bytes and + // `CRU_CLKGATE_CON14` (0x838) is within the first page. One write-masked + // MMIO store; the gate register tolerates concurrent write-masked updates to + // other bits, so no read-modify-write is needed. + unsafe { + cru.as_ptr() + .add(CRU_CLKGATE_CON14) + .cast::() + .write_volatile(SPI2_CLK_UNGATE); + } + info!( + "pmic_spi: ungated SPI2 PCLK+CLK (CRU {:#x} <= {SPI2_CLK_UNGATE:#010x})", + RK3588_CRU_BASE + CRU_CLKGATE_CON14 + ); + true +} + +/// Release the SPI2 controller from soft-reset in the main CRU so its shift +/// engine can transact. De-assert only (write masked zeros to SRST_P_SPI2 bit 8 + +/// SRST_SPI2 bit 13); we never assert, so nothing else is disturbed, and the +/// BAUDR/CTRLR0 that `configure` (re)writes afterwards take effect on a released +/// core. Idempotent (re-clearing an already-released reset is a no-op). Returns +/// `false` only if the CRU cannot be mapped. +fn deassert_spi2_reset() -> bool { + let cru = match iomap(RK3588_CRU_BASE, RK3588_CRU_SIZE) { + Ok(v) => v, + Err(err) => { + warn!("pmic_spi: iomap CRU {RK3588_CRU_BASE:#x} failed: {err:?}"); + return false; + } + }; + // SAFETY: the CRU mapping covers `RK3588_CRU_SIZE` bytes and + // `CRU_SOFTRST_CON14` (0xa38) is within the first page. One write-masked MMIO + // store clearing only the two SPI2 reset bits. + unsafe { + cru.as_ptr() + .add(CRU_SOFTRST_CON14) + .cast::() + .write_volatile(SPI2_RST_DEASSERT); + } + axklib::time::busy_wait(Duration::from_micros(RST_SETTLE_US)); + info!( + "pmic_spi: de-asserted SPI2 P+core reset (CRU {:#x} <= {SPI2_RST_DEASSERT:#010x})", + RK3588_CRU_BASE + CRU_SOFTRST_CON14 + ); + true +} + +/// Apply the FULL `spi@feb20000` pinctrl-0 (mux **+ pull + drive-strength + +/// schmitt**) via the registered Rockchip pinctrl driver — the same path that +/// fixed i2c0. +/// +/// Board finding (boot #7): our previous mux-only check saw "already func1" yet +/// read AND write to the RK806 were both dead. i2c0 had the identical symptom and +/// was fixed by applying the *full* pin config, not just the mux — the DTS +/// pinconf flags (pull/drive/slew on `spi2m2_pins`/`spi2m2_cs0`) were u-boot's or +/// default, not what the RK806 link needs, so CLK/MOSI/CS never cleanly reached +/// the chip. Rather than hand-decode the pinconf, we apply the node's `pinctrl-0` +/// state through the in-tree pinctrl driver, which already encodes it. Pad-config +/// change only (no PMIC access); best-effort — any failure is logged and the +/// later transaction times out into a safe no-op. +fn set_spi2_pinmux() { + let Some(pinctrl) = rdrive::get_one::() else { + warn!("pmic_spi: PinctrlDevice not registered; cannot apply spi2 pinctrl"); + return; + }; + let mut pinctrl = match pinctrl.lock() { + Ok(guard) => guard, + Err(err) => { + warn!("pmic_spi: failed to lock PinctrlDevice: {err}; cannot apply spi2 pinctrl"); + return; + } + }; + let Some(fdt) = rdrive::with_fdt(Clone::clone) else { + warn!("pmic_spi: live FDT not found; cannot apply spi2 pinctrl"); + return; + }; + // Match the spi2 controller node by reg base (several SPI controllers share + // the rk3066-spi compatible). + let node = fdt + .find_compatible(&["rockchip,rk3066-spi"]) + .into_iter() + .find(|n| { + n.regs() + .into_iter() + .next() + .is_some_and(|r| r.address as usize == RK3588_SPI2_BASE) + }); + let Some(node) = node else { + warn!("pmic_spi: spi2 node @ {RK3588_SPI2_BASE:#x} not found in FDT; cannot apply pinctrl"); + return; + }; + match pinctrl.apply_fdt_default_state(&fdt, node.as_node()) { + Ok(()) => info!( + "pmic_spi: applied spi2 pinctrl-0 (spi2m2 clk/mosi/miso/cs0 mux+pull+drive) via \ + rockchip pinctrl" + ), + Err(err) => warn!("pmic_spi: failed to apply spi2 pinctrl-0: {err:?}"), + } + + // MISO loopback probe (last untested register): force GPIO0_B3 (= SPI2_MISO) to + // INPUT direction. Linux leaves it input (GPIO0 DDR_L bit11 = 0); if StarryOS's + // boot left B3 as a GPIO *output*, the pad would drive instead of sample, so the + // controller reads its own TX shift register — exactly the observed rx==tx + // loopback. The func-1 mux normally overrides GPIO direction, so this is a + // long-shot, but it is the one register we have not yet forced. GPIO0 @ 0xfd8a0000, + // SWPORT_DR_L=0x00, SWPORT_DDR_L=0x08 (write-masked: hi16 = per-bit mask). + if let Ok(gpio0) = iomap(0xfd8a_0000, 0x100) { + let p = gpio0.as_ptr(); + // SAFETY: GPIO0 mapping covers 0x100; DDR_L is at 0x08, within the page. + let ddr = unsafe { p.add(0x08).cast::().read_volatile() }; + info!( + "pmic_spi: GPIO0 DDR_L={:#010x} B3(bit11)={} (0=input) — forcing B3 input", + ddr, + (ddr >> 11) & 1 + ); + // Force B3 -> input: mask bit11 (hi16), value 0 (lo16). + unsafe { p.add(0x08).cast::().write_volatile((1u32 << 11) << 16) }; + } +} + +/// Read-only dump of the applied GPIO0 A/B pad config (boot #9 loopback debug). +/// Logs the whole PMU1_IOC first page for a direct line-by-line diff against +/// Linux's known-good reference, plus the decoded B3/MISO fields (mux 0x08[15:12], +/// drive 0x18[15:12], pull 0x24[7:6], smt 0x34[3]). Linux (works): B3 mux=1, +/// pull=3 (up), smt=0. Whichever field differs on our side is the loopback cause. +/// No writes. +fn dump_spi2_iomux() { + let ioc = match iomap(RK3588_IOC_BASE, RK3588_IOC_SIZE) { + Ok(v) => v.as_ptr(), + Err(err) => { + warn!("pmic_spi: iomap IOC {RK3588_IOC_BASE:#x} failed: {err:?}; skipping IOC dump"); + return; + } + }; + // SAFETY: the IOC mapping covers `RK3588_IOC_SIZE` bytes; every offset read + // below is < 0x44, well within the first page. Read-only. + let rd = |off: usize| unsafe { ioc.add(off).cast::().read_volatile() }; + let (mux_b, drv_b, pull_b, smt_b) = (rd(0x08), rd(0x18), rd(0x24), rd(0x34)); + info!( + "pmic_spi: PMU1_IOC 00={:08x} 04={:08x} 08={:08x} 0c={:08x} 10={:08x} 14={:08x} 18={:08x} \ + 1c={:08x} 20={:08x} 24={:08x} 28={:08x} 2c={:08x} 30={:08x} 34={:08x} 40={:08x}", + rd(0x00), + rd(0x04), + rd(0x08), + rd(0x0c), + rd(0x10), + rd(0x14), + rd(0x18), + rd(0x1c), + rd(0x20), + rd(0x24), + rd(0x28), + rd(0x2c), + rd(0x30), + rd(0x34), + rd(0x40), + ); + info!( + "pmic_spi: B3/MISO decoded: mux={} (want 1) drive={} pull={} (want 3) smt={} (want 0)", + (mux_b >> 12) & 0xf, + (drv_b >> 12) & 0xf, + (pull_b >> 6) & 0x3, + (smt_b >> 3) & 0x1, + ); +} + +/// Map and initialize the RK806-on-SPI2 controller. Idempotent; safe to call +/// from a `PostKernel` probe context (MMU up). +/// +/// Ungates the SPI2 clocks (U-Boot gates them after talking to the RK806 at +/// boot), maps + configures the controller, then verifies it actually responds +/// on its APB. Returns `true` when the controller is bound and ready — including +/// an idempotent re-`init()` of an already-bound controller. Returns `false` on +/// a CRU/SPI2 MMIO mapping failure **or** if the controller does not respond +/// after ungate (still gated, or held in soft-reset — which would need a +/// separate reset de-assert). In every `false` case the controller is left +/// **unbound**, so `get_uv`/`set_uv*` are safe no-ops that leave the boot +/// voltage rather than returning a fabricated `0`. +pub fn init() -> bool { + let mut guard = PMIC.lock(); + if guard.is_some() { + // Already bound: the controller is ready, so report success (not a + // failure) and keep the existing binding. + return true; + } + if !ungate_spi2_clocks() { + return false; + } + let virt = match iomap(RK3588_SPI2_BASE, RK3588_SPI2_SIZE) { + Ok(v) => v, + Err(err) => { + warn!("pmic_spi: iomap {RK3588_SPI2_BASE:#x} failed: {err:?}"); + return false; + } + }; + // SAFETY: `iomap` returned a mapping of `RK3588_SPI2_SIZE` bytes at the SPI2 + // controller base; `base` is only ever used for volatile MMIO within that + // window, serialized through the `PMIC` mutex. + let dev = Rk806Spi { + base: virt.as_ptr(), + }; + // Release the controller core from soft-reset (boot #2: is_alive passed but + // the shift engine read all-zero — core reset held). Must precede configure() + // so BAUDR/CTRLR0 land on a released core. + if !deassert_spi2_reset() { + return false; + } + // Apply the FULL spi2 pinctrl-0 (mux + pull + drive + schmitt) via the + // rockchip pinctrl driver — boot #7 showed a mux-only match isn't enough; the + // pad config is what fixed i2c0. Must precede any transfer. + set_spi2_pinmux(); + // Boot #8: read confirmed MISO loopback. Dump the applied SPI2 mux so we can + // check B3(MISO) is really func1 (vs A6/MOSI which works). + dump_spi2_iomux(); + dev.configure(); + if !dev.is_alive() { + warn!( + "pmic_spi: SPI2 controller not responding after ungate (CTRLR0 read-back failed; APB \ + may still be gated or held in reset); leaving A55 rail untouched" + ); + return false; + } + // Read-only reachability probe (chip-ID + raw DCDC2 frame). + dev.log_diagnostics(); + // Boot #8: the read now returns data but RX mirrors TX — distinguish MISO + // loopback from a value-position offset (read-only). + dev.probe_read_shape(); + info!("pmic_spi: RK806/SPI2 bound at {RK3588_SPI2_BASE:#x}"); + *guard = Some(dev); + true +} + +/// Read the current `vdd_cpu_lit` (A55 rail) voltage in microvolts, or `None` +/// if the controller is not initialized or the SPI read timed out. +pub fn get_uv() -> Option { + let guard = PMIC.lock(); + let dev = guard.as_ref()?; + let sel = dev.rk806_read(RK806_BUCK2_ON_VSEL)?; + let uv = vsel_to_uv(sel); + info!("pmic_spi: A55 vdd_cpu_lit = {uv} uV (buck2 vsel {sel:#04x})"); + Some(uv) +} + +/// Set the A55 rail to `target_uv` in a single write, **down only**. +/// +/// Refuses (returns `false`, no write) when: not initialized; the SPI read of +/// the boot voltage fails; `target_uv` is below [`A55_MIN_UV`]; or `target_uv` +/// is above the boot voltage (never raises). On an accepted target it encodes +/// the selector, writes buck-2 `ON_VSEL`, reads it back, and returns `true` only +/// when the read-back selector matches. +// Down-only single-write A55 setter; retained as part of the PMIC-SPI API surface +// for the calibration/voltage-lever path even when the shipped governor build does +// not call it. +#[allow(dead_code)] +pub fn set_uv(target_uv: u32) -> bool { + let guard = PMIC.lock(); + let Some(dev) = guard.as_ref() else { + warn!("pmic_spi: set_uv before init; ignored"); + return false; + }; + let Some(boot_sel) = dev.rk806_read(RK806_BUCK2_ON_VSEL) else { + warn!("pmic_spi: set_uv could not read boot voltage; leaving rail untouched"); + return false; + }; + let boot_uv = vsel_to_uv(boot_sel); + let Some(target_sel) = check_down_only(target_uv, boot_uv) else { + return false; + }; + write_and_verify(dev, target_sel) +} + +/// Lower the A55 rail to `target_uv` in `<=`[`MAX_STEP_UV`] steps, verifying +/// each write and settling between steps, **down only**. +/// +/// Same refusal rules as [`set_uv`]. A `target_uv` equal to the current voltage +/// is an accepted no-op (`true`). Any read-back mismatch aborts mid-descent and +/// returns `false`, leaving the last verified selector in place. +pub fn set_uv_stepped(target_uv: u32) -> bool { + let guard = PMIC.lock(); + let Some(dev) = guard.as_ref() else { + warn!("pmic_spi: set_uv_stepped before init; ignored"); + return false; + }; + let Some(cur_sel) = dev.rk806_read(RK806_BUCK2_ON_VSEL) else { + warn!("pmic_spi: set_uv_stepped could not read current voltage; leaving rail untouched"); + return false; + }; + let cur_uv = vsel_to_uv(cur_sel); + let Some(target_sel) = check_down_only(target_uv, cur_uv) else { + return false; + }; + + // Higher selector == higher voltage. Down-only means target_sel <= cur_sel. + let step_sel = (MAX_STEP_UV / RK806_VSEL_STEP_UV).max(1) as u8; + let mut sel = cur_sel; + while sel > target_sel { + let next = target_sel.max(sel.saturating_sub(step_sel)); + if !write_and_verify(dev, next) { + return false; + } + axklib::time::busy_wait(Duration::from_micros(STEP_SETTLE_US)); + sel = next; + } + info!( + "pmic_spi: A55 vdd stepped to {} uV (vsel {sel:#04x})", + vsel_to_uv(sel) + ); + true +} + +/// **Diagnostic** force-write of the A55 rail (DCDC2 `ON_VSEL`) — proves whether +/// SPI *writes* physically reach the RK806 while reads are dead. Not part of the +/// normal DVFS API; intended to be called once from a cpufreq test flag. +/// +/// Safety: hard-clamped to `[675_000, 950_000]` uV — the A55 (little cluster) OPP +/// voltage range (675 mV @ 1008 MHz up to 950 mV @ 1800 MHz, all Linux-proven +/// freq/voltage pairs). The governor only ever passes OPP-matched voltages, so any +/// accepted value is a proven-safe rail voltage regardless of the (currently +/// unreadable) present value, and RK3588's voltage-coupled clock tracks the rail in +/// lockstep. Unlike [`set_uv`] it deliberately SKIPS the read-current/down-only +/// guard — the read path is a scope-wall (rx==tx loopback) and the write is what we +/// have. If the SPI path can't reach the RK806, the write is simply a no-op. +/// +/// This is the A55 voltage-set primitive for the ondemand governor (the read-back +/// path is deferred pending the MISO scope fix). Attempts a read-back for the log +/// (`0x00` while reads fail) and returns the write transfer's success. +pub fn force_write_dcdc2(target_uv: u32) -> bool { + if !(675_000..=950_000).contains(&target_uv) { + warn!( + "pmic_spi: force_write_dcdc2 refused target {target_uv} uV (outside A55 OPP range \ + [675000, 950000])" + ); + return false; + } + let guard = PMIC.lock(); + let Some(dev) = guard.as_ref() else { + warn!("pmic_spi: force_write_dcdc2 before init; ignored"); + return false; + }; + let Some(sel) = uv_to_vsel(target_uv) else { + warn!("pmic_spi: force_write_dcdc2 target {target_uv} uV not encodable"); + return false; + }; + let wrote = dev.rk806_write(RK806_BUCK2_ON_VSEL, sel); + let readback = dev.rk806_read(RK806_BUCK2_ON_VSEL); + info!( + "pmic_spi: force_write_dcdc2: wrote vsel={sel:#04x} ({target_uv} uV) xfer_ok={wrote} \ + readback={readback:#04x?}" + ); + wrote +} + +/// Shared down-only clamp: returns the encoded target selector, or `None` +/// (after logging why) when the target must be refused. +fn check_down_only(target_uv: u32, reference_uv: u32) -> Option { + if target_uv < A55_MIN_UV { + warn!("pmic_spi: refusing target {target_uv} uV below floor {A55_MIN_UV} uV"); + return None; + } + if target_uv > reference_uv { + warn!("pmic_spi: refusing to RAISE vdd ({reference_uv} uV -> {target_uv} uV); down-only"); + return None; + } + match uv_to_vsel(target_uv) { + Some(sel) => Some(sel), + None => { + warn!("pmic_spi: target {target_uv} uV not encodable in buck range 1"); + None + } + } +} + +/// Write buck-2 `ON_VSEL` and confirm the read-back selector matches. +fn write_and_verify(dev: &Rk806Spi, sel: u8) -> bool { + if !dev.rk806_write(RK806_BUCK2_ON_VSEL, sel) { + warn!("pmic_spi: buck2 vsel write timed out (sel {sel:#04x})"); + return false; + } + match dev.rk806_read(RK806_BUCK2_ON_VSEL) { + Some(rb) if rb == sel => true, + Some(rb) => { + warn!("pmic_spi: buck2 vsel read-back {rb:#04x} != written {sel:#04x}; aborting"); + false + } + None => { + warn!("pmic_spi: buck2 vsel read-back failed after write; aborting"); + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vsel_decodes_known_opp_nominals() { + assert_eq!(vsel_to_uv(0x1c), 675_000); // sel 28 + assert_eq!(vsel_to_uv(0x28), 750_000); // sel 40 + assert_eq!(vsel_to_uv(0x30), 800_000); // sel 48 + assert_eq!(vsel_to_uv(0), 500_000); + // Selector 159 (range-1 max) is 1_493_750 uV; 1_500_000 is selector 160. + assert_eq!(vsel_to_uv(RK806_VSEL_R1_MAX_SEL), 1_493_750); + } + + #[test] + fn vsel_decodes_upper_ranges() { + assert_eq!(vsel_to_uv(160), 1_500_000); // range 2 base + assert_eq!(vsel_to_uv(161), 1_525_000); + assert_eq!(vsel_to_uv(235), 3_375_000); + assert_eq!(vsel_to_uv(236), 3_400_000); // range 3 fixed + assert_eq!(vsel_to_uv(255), 3_400_000); + } + + #[test] + fn uv_encodes_opp_nominals_exactly() { + assert_eq!(uv_to_vsel(675_000), Some(0x1c)); + assert_eq!(uv_to_vsel(750_000), Some(0x28)); + assert_eq!(uv_to_vsel(800_000), Some(0x30)); + } + + #[test] + fn uv_encode_roundtrips_within_range1() { + for sel in 0..=RK806_VSEL_R1_MAX_SEL { + let uv = vsel_to_uv(sel); + assert_eq!(uv_to_vsel(uv), Some(sel), "sel {sel}"); + } + } + + #[test] + fn uv_encode_rejects_out_of_range1() { + assert_eq!(uv_to_vsel(499_000), None); + assert_eq!(uv_to_vsel(1_600_000), None); + } + + #[test] + fn uv_encode_rejects_unaligned_steps() { + // In-range but not on a 6.25 mV boundary: reject rather than approximate. + assert_eq!(uv_to_vsel(675_000 + 1), None); + assert_eq!(uv_to_vsel(676_000), None); // 176_000 % 6250 != 0 + assert_eq!(uv_to_vsel(500_001), None); + // Boundaries stay accepted. + assert_eq!(uv_to_vsel(500_000), Some(0)); + assert_eq!(uv_to_vsel(506_250), Some(1)); + } + + #[test] + fn down_only_clamp_refuses_below_floor_and_raises() { + // Below the 675 mV floor. + assert_eq!(check_down_only(650_000, 800_000), None); + // Raise attempt (target above reference). + assert_eq!(check_down_only(825_000, 800_000), None); + // Legal down move. + assert_eq!(check_down_only(675_000, 800_000), Some(0x1c)); + // No-op (equal) is allowed. + assert_eq!(check_down_only(800_000, 800_000), Some(0x30)); + } + + #[test] + fn baud_divider_even_and_under_target() { + // 200 MHz functional clock, 1 MHz target -> even divisor, sclk <= 1 MHz. + let div = baud_divider(200_000_000, 1_000_000); + assert_eq!(div % 2, 0); + assert!(200_000_000 / div <= 1_000_000, "div {div}"); + assert_eq!(div, 200); + } + + #[test] + fn baud_divider_clamps_and_stays_even() { + assert_eq!(baud_divider(1_000_000, 1_000_000) % 2, 0); + assert!(baud_divider(u32::MAX, 1) <= BAUDR_MAX); + assert!(baud_divider(1, 1_000_000) >= BAUDR_MIN); + } + + #[test] + fn step_size_is_four_selectors() { + assert_eq!(MAX_STEP_UV / RK806_VSEL_STEP_UV, 4); // 25 mV / 6.25 mV + } + + #[test] + fn spi2_ungate_value_matches_gate_bits() { + // clkgate_con(14): PCLK_SPI2 = bit 8, CLK_SPI2 = bit 13. Write-masked: + // high half selects the bits, low half writes 0 to enable them. + assert_eq!(SPI2_CLK_UNGATE, 0x2100_0000); + assert_eq!(SPI2_CLK_UNGATE & 0xffff, 0); // low half all-0 => enable + assert_eq!(SPI2_CLK_UNGATE >> 16, (1 << 8) | (1 << 13)); + assert_eq!(CRU_CLKGATE_CON14, 14 * 4 + 0x800); // 0x838 + } + + #[test] + fn ctrlr0_mode0_8bit_value() { + // Linux spi-rockchip.c live value on this board: master, mode 0, 8-bit + // DFS, one-cycle CS delay, EM_BIG, APB 8-bit access. + assert_eq!(SPI_CTRLR0_MODE0_8BIT, 0x2c01); + } + + #[test] + fn spi2_reset_deassert_matches_reset_bits() { + // softrst_con(14): SRST_P_SPI2 = bit 8, SRST_SPI2 = bit 13. Write-masked + // de-assert: high half selects the bits, low half writes 0 to release. + assert_eq!(SPI2_RST_DEASSERT, 0x2100_0000); + assert_eq!(SPI2_RST_DEASSERT & 0xffff, 0); // low half all-0 => released + assert_eq!(SPI2_RST_DEASSERT >> 16, (1 << 8) | (1 << 13)); + assert_eq!(CRU_SOFTRST_CON14, 14 * 4 + 0xa00); // 0xa38 + } +} diff --git a/drivers/ax-driver/src/soc/scmi.rs b/drivers/ax-driver/src/soc/scmi.rs index c9586aa7c4..67edf34876 100644 --- a/drivers/ax-driver/src/soc/scmi.rs +++ b/drivers/ax-driver/src/soc/scmi.rs @@ -162,6 +162,44 @@ pub fn set_clock_rate(_phandle: Phandle, clock_id: u32, rate: u64) -> Option<()> } } +/// Query the rates the platform permits for `clock_id` +/// (SCMI `CLOCK_DESCRIBE_RATES`, message 0x4). +/// +/// This is a **read-only** operation: it changes no clock state and is intended +/// as a safety preflight to confirm the firmware actually services a clock +/// before any rate is programmed. Returns `Some(())` when the platform answers +/// the query (the clock exists and its operations are serviced) and `None` when +/// it rejects it. The permitted rates are logged for diagnostics. `_phandle` is +/// ignored (single global agent), mirroring the other helpers here. +pub fn describe_rates(_phandle: Phandle, clock_id: u32) -> Option<()> { + if !SCMI_REGISTERED.load(Ordering::Acquire) { + warn!( + "SCMI describe rates requested before SCMI registration: clock_id={:#x}", + clock_id + ); + return None; + } + let mut guard = SCMI.lock(); + let scmi = guard.as_mut()?; + let mut clock = scmi.protocol_clk_no_init(); + match clock.describe_rates(clock_id, 0) { + Ok(rates) => { + info!( + "SCMI describe rates: clock_id={:#x}, rates={:?}", + clock_id, rates + ); + Some(()) + } + Err(err) => { + warn!( + "SCMI describe rates failed: clock_id={:#x}, {:?}", + clock_id, err + ); + None + } + } +} + struct ScmiDevice; impl DriverGeneric for ScmiDevice { diff --git a/os/StarryOS/kernel/src/entry.rs b/os/StarryOS/kernel/src/entry.rs index ae0363d9fe..5d2823d986 100644 --- a/os/StarryOS/kernel/src/entry.rs +++ b/os/StarryOS/kernel/src/entry.rs @@ -29,6 +29,14 @@ pub fn init(args: &[String], envs: &[String]) { pseudofs::mount_all().expect("Failed to mount pseudofs"); spawn_alarm_task(); + // DVFS: a one-shot OPP-calibration boot runs the sweep and skips the governor; + // otherwise start the ondemand governor. Both run here (early init, before the + // console tty handoff) so their kernel logs reach the serial console. + if ax_driver::cpufreq::calibrate_wanted() { + run_opp_calibration(); + } else { + spawn_cpufreq_governor(); + } pseudofs::usbfs::start_event_pump(); ax_alloc::register_page_reclaim_fn(ax_fs_ng::vfs::page_cache_reclaim); @@ -118,3 +126,68 @@ pub fn init(args: &[String], envs: &[String]) { .flush() .expect("Failed to flush rootfs"); } + +/// Run the one-shot DVFS OPP calibration sweep (gated by the driver's `CALIBRATE` +/// const). Each cluster's (voltage x ring) sweep must execute ON a core of that +/// cluster to read that core's own PMU cycle counter, so we pin a task per cluster +/// (cpu0=A55, cpu4=A76 big0, cpu6=A76 big1) via `set_current_affinity` and run +/// them sequentially (the two A76 rails share one I2C bus). Synchronous: it blocks +/// init briefly so the `CAL` log lines land before the console tty handoff. +fn run_opp_calibration() { + info!("cpufreq: running OPP calibration sweep (governor disabled this boot)"); + for &(cluster_idx, cpu) in &[(0usize, 0usize), (1, 4), (2, 6)] { + let task = ax_task::spawn_raw( + move || { + ax_task::set_current_affinity(ax_task::AxCpuMask::one_shot(cpu)); + ax_driver::cpufreq::calibrate_cluster(cluster_idx, cpu); + }, + String::from("cpufreq-cal"), + ax_task::default_task_stack_size(), + ); + task.join(); + } + info!("cpufreq: OPP calibration sweep complete"); +} + +/// Start the CPU DVFS ondemand governor. +/// +/// The frequency/voltage policy and the SCMI+PMIC apply live in the cpufreq +/// driver (`ax_driver::cpufreq`); this kernel task is only the driver's periodic +/// *loop*. The loop must live here, not in the driver, because ax-driver sits +/// below ax-task/ax-hal in the dependency graph (they pull ax-driver back in via +/// axplat-dyn), so spawning a task inside the driver would be a cyclic dep. Each +/// period we snapshot the per-CPU busy counters the scheduler tick maintains and +/// hand them to `governor_poll`, which decides and applies any OPP change. +/// +/// No-op unless the driver armed the governor (feature on and both CPU-rail PMIC +/// buses up); otherwise every cluster stays on its boot OPP. +fn spawn_cpufreq_governor() { + if !ax_driver::cpufreq::governor_wanted() { + return; + } + info!("Initialize cpufreq ondemand governor..."); + ax_task::spawn_raw( + cpufreq_governor_loop, + String::from("cpufreq-gov"), + ax_task::default_task_stack_size(), + ); +} + +/// Periodic body of the DVFS governor task: sleep, sample every CPU's cumulative +/// busy-tick counter, and let the driver scale each cluster to match load. The +/// slow work (SCMI SMC + PMIC I2C/SPI voltage ramp) happens inside +/// `governor_poll`, which is why this runs in a sleepable task rather than the +/// scheduler tick. +fn cpufreq_governor_loop() { + let period = core::time::Duration::from_millis(ax_driver::cpufreq::governor_period_ms()); + loop { + ax_task::sleep(period); + // RK3588 has 8 CPUs; an offline core's counter never advances, so it + // simply reads as idle (conservative — never over-scales). + let mut busy = [0u64; 8]; + for (cpu, slot) in busy.iter_mut().enumerate() { + *slot = ax_task::cpu_busy_ticks(cpu); + } + ax_driver::cpufreq::governor_poll(&busy); + } +} diff --git a/os/arceos/modules/axtask/src/api.rs b/os/arceos/modules/axtask/src/api.rs index d44f9960e8..399d0f8dee 100644 --- a/os/arceos/modules/axtask/src/api.rs +++ b/os/arceos/modules/axtask/src/api.rs @@ -199,6 +199,19 @@ pub fn next_timer_deadline_nanos() -> Option { crate::timers::next_deadline_nanos() } +/// Scheduler ticks CPU `cpu` has spent running a non-idle task since boot. +/// +/// This is the load metric for an ondemand cpufreq governor: a monotonic per-CPU +/// counter bumped once per timer tick when the CPU is not idle. Sample the delta +/// over a window and divide by the elapsed ticks to get the busy fraction. Returns +/// 0 for an out-of-range `cpu`. Requires the `irq` feature to actually advance +/// (the counter only moves inside the timer tick). +pub fn cpu_busy_ticks(cpu: usize) -> u64 { + crate::run_queue::BUSY_TICKS + .get(cpu) + .map_or(0, |t| t.load(core::sync::atomic::Ordering::Relaxed)) +} + #[cfg(feature = "irq")] #[doc(hidden)] pub fn note_programmed_timer_deadline_nanos(deadline_nanos: u64) { diff --git a/os/arceos/modules/axtask/src/run_queue.rs b/os/arceos/modules/axtask/src/run_queue.rs index b5c839028a..71be34c54d 100644 --- a/os/arceos/modules/axtask/src/run_queue.rs +++ b/os/arceos/modules/axtask/src/run_queue.rs @@ -58,6 +58,15 @@ static mut RUN_QUEUES: [MaybeUninit<&'static mut AxRunQueue>; crate::build_info: #[allow(clippy::declare_interior_mutable_const)] // It's ok because it's used only for initialization `RUN_QUEUES`. const ARRAY_REPEAT_VALUE: MaybeUninit<&'static mut AxRunQueue> = MaybeUninit::uninit(); +/// Per-CPU count of scheduler ticks during which a non-idle task was running, for +/// the ondemand cpufreq governor's load metric. Bumped once per timer tick in +/// [`AxRunQueue::scheduler_timer_tick`] when the current task is not the idle task +/// (that path already runs with IRQ + preempt disabled). Read cross-CPU by the +/// governor via [`crate::cpu_busy_ticks`]; `Relaxed` atomics keep the bump a single +/// instruction on the owning CPU while avoiding a data race on the read. +pub(crate) static BUSY_TICKS: [core::sync::atomic::AtomicU64; crate::build_info::CPU_CAPACITY] = + [const { core::sync::atomic::AtomicU64::new(0) }; crate::build_info::CPU_CAPACITY]; + #[cfg(not(feature = "host-test"))] fn main_task_stack() -> TaskStack { let (stack_ptr, stack_size) = ax_hal::mem::boot_stack_bounds(this_cpu_id()); @@ -640,9 +649,17 @@ impl CurrentRunQueueRef<'_, G> { #[cfg(feature = "irq")] pub fn scheduler_timer_tick(&mut self) { let curr = &self.current_task; - if !curr.is_idle() && self.inner.scheduler.lock().task_tick(curr) { - #[cfg(feature = "preempt")] - curr.set_preempt_pending(true); + if !curr.is_idle() { + // Ondemand-governor load accounting: this CPU ran a real (non-idle) + // task this tick. Already IRQ + preempt off here; a single relaxed + // fetch_add is essentially free. + if let Some(t) = BUSY_TICKS.get(this_cpu_id()) { + t.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + } + if self.inner.scheduler.lock().task_tick(curr) { + #[cfg(feature = "preempt")] + curr.set_preempt_pending(true); + } } } From a93e784fc7ac9d1a0dcae2ac4380d6fa783bf7d3 Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Tue, 21 Jul 2026 11:27:17 +0800 Subject: [PATCH 02/10] fix(cpufreq): gate aarch64 PMU inline asm behind cfg(target_arch) with host stubs --- drivers/ax-driver/src/soc/rockchip/cpufreq.rs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs index 60c2fbd579..12c63f6144 100644 --- a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs +++ b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs @@ -708,30 +708,67 @@ pub fn calibrate_wanted() -> bool { CALIBRATE && GOV_READY.load(Ordering::Acquire) } +// The calibration sweep is board-only (gated off by `CALIBRATE = false` in +// production), but `rk3588-cpufreq` is a public feature and must still COMPILE +// on a non-aarch64 host (e.g. a plain `cargo build`/`cargo test` on the CI +// host). These four leaf reads are the only AArch64 inline asm in this module; +// everything above them (`measure_mhz`, `cal_delay_ms`, `calibrate_cluster`) +// is arch-generic and calls only these, so gating just the leaves keeps the +// aarch64/board behavior byte-for-byte identical while giving every other +// target a harmless stub (0 never causes a hang: `cal_delay_ms`'s `frq.max(1)` +// and `measure_mhz`'s zero-length window both degenerate to an immediate +// return rather than spinning). +#[cfg(target_arch = "aarch64")] #[inline] fn rd_pmccntr() -> u64 { let v: u64; unsafe { core::arch::asm!("mrs {}, pmccntr_el0", out(reg) v) }; v } +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn rd_pmccntr() -> u64 { + 0 +} + +#[cfg(target_arch = "aarch64")] #[inline] fn rd_cntvct() -> u64 { let v: u64; unsafe { core::arch::asm!("mrs {}, cntvct_el0", out(reg) v) }; v } +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn rd_cntvct() -> u64 { + 0 +} + +#[cfg(target_arch = "aarch64")] #[inline] fn rd_cntfrq() -> u64 { let v: u64; unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) v) }; v } +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn rd_cntfrq() -> u64 { + 0 +} + +#[cfg(target_arch = "aarch64")] #[inline] fn rd_mpidr() -> u64 { let v: u64; unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) v) }; v } +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn rd_mpidr() -> u64 { + 0 +} /// Busy-wait `ms` milliseconds against the fixed-rate `CNTVCT` clock. fn cal_delay_ms(ms: u64) { From 43f401c38232354a327d1e51dab193f8cd3df82e Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Tue, 21 Jul 2026 11:31:53 +0800 Subject: [PATCH 03/10] fix(cpufreq): make OPP apply transactional so a failed voltage/clock write does not commit IDX --- drivers/ax-driver/src/soc/rockchip/cpufreq.rs | 114 +++++++++++++++--- 1 file changed, 98 insertions(+), 16 deletions(-) diff --git a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs index 12c63f6144..7b0a5285e5 100644 --- a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs +++ b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs @@ -510,16 +510,79 @@ impl Cluster { /// the voltage-coupled clock never overshoots its rail: /// - going UP: raise voltage first, then the SCMI ring (clock follows up); /// - going DOWN: lower the SCMI ring first, then voltage (clock follows down). -fn apply_opp(cluster: Cluster, opp: Opp, going_up: bool) { +/// +/// TRANSACTIONAL: stops at the first failed step instead of falling through to +/// the second (which could otherwise create exactly the freq-high/voltage-low +/// window this ordering exists to prevent), and returns `true` only when BOTH +/// steps are confirmed. The caller ([`governor_poll`]) must commit its software +/// OPP index only on `true`; on `false` it must keep tracking the last +/// confirmed index so the next poll retries from there. +/// +/// No-undervolt argument (both failure points, each direction): +/// - UPSHIFT, voltage step fails: the ring set is skipped entirely, so +/// nothing changed — old freq + old voltage, still a valid, previously +/// confirmed pairing. +/// - UPSHIFT, voltage step succeeds but the ring set fails: the rail is +/// already at (or above) `opp.uv` while the ring is still at its old, +/// lower value — over-volted for whatever it is currently delivering, +/// never under-volted. +/// - DOWNSHIFT, ring step fails: the voltage lower is skipped entirely, so +/// nothing changed — old freq + old voltage, still valid. +/// - DOWNSHIFT, ring step succeeds but the voltage step fails: the ring is +/// already at its new, lower value while the rail is still at its old, +/// higher voltage — over-volted for the new (lower) clock, never +/// under-volted. +#[must_use] +fn apply_opp(cluster: Cluster, opp: Opp, going_up: bool) -> bool { let phandle = Phandle::from(0u32); let hz = opp.ring_khz as u64 * 1_000; if going_up { - cluster.set_voltage(opp.uv); - scmi::set_clock_rate(phandle, cluster.clock_id(), hz); + if !cluster.set_voltage(opp.uv) { + warn!( + "cpufreq: {} upshift to {} mV failed; leaving clock id {} unchanged (no \ + undervolt: old freq stays paired with old voltage)", + cluster.name(), + opp.uv / 1_000, + cluster.clock_id() + ); + return false; + } + if scmi::set_clock_rate(phandle, cluster.clock_id(), hz).is_none() { + warn!( + "cpufreq: {} voltage already raised to {} mV but SCMI rejected clock id {} -> {} \ + Hz; not committing this OPP (safe: over-volted for the old, lower clock, never \ + under-volted)", + cluster.name(), + opp.uv / 1_000, + cluster.clock_id(), + hz + ); + return false; + } } else { - scmi::set_clock_rate(phandle, cluster.clock_id(), hz); - cluster.set_voltage(opp.uv); + if scmi::set_clock_rate(phandle, cluster.clock_id(), hz).is_none() { + warn!( + "cpufreq: {} downshift: SCMI rejected clock id {} -> {} Hz; leaving voltage \ + unchanged (no undervolt: old freq stays paired with old voltage)", + cluster.name(), + cluster.clock_id(), + hz + ); + return false; + } + if !cluster.set_voltage(opp.uv) { + warn!( + "cpufreq: {} clock already lowered to {} Hz but voltage write to {} mV failed; \ + not committing this OPP (safe: over-volted for the new, lower clock, never \ + under-volted)", + cluster.name(), + hz, + opp.uv / 1_000 + ); + return false; + } } + true } /// Period, in ms, the kernel governor task sleeps between [`governor_poll`]s. @@ -640,17 +703,36 @@ pub fn governor_poll(busy: &[u64]) { }; if new != cur { - apply_opp(cluster, opps[new], new > cur); - IDX[ci].store(new, Ordering::Relaxed); - info!( - "gov: {} peak={}% opp {}->{} = {} MHz @ {} mV", - cluster.name(), - peak_pct, - cur, - new, - opps[new].mhz, - opps[new].uv / 1_000 - ); + // Only commit IDX (the software record of the last CONFIRMED OPP) + // when both the voltage write and the SCMI clock set are verified + // successful. On failure, IDX is left at `cur` — the hardware is + // always left in a safe (never-undervolted, see `apply_opp`) state + // for that index, so the next poll retries the same climb/descent + // from a known-good starting point rather than silently pretending + // the change took effect. + if apply_opp(cluster, opps[new], new > cur) { + IDX[ci].store(new, Ordering::Relaxed); + info!( + "gov: {} peak={}% opp {}->{} = {} MHz @ {} mV", + cluster.name(), + peak_pct, + cur, + new, + opps[new].mhz, + opps[new].uv / 1_000 + ); + } else { + warn!( + "gov: {} peak={}% opp {}->{} FAILED to apply; staying at {} ({} MHz @ {} mV)", + cluster.name(), + peak_pct, + cur, + new, + cur, + opps[cur].mhz, + opps[cur].uv / 1_000 + ); + } } } } From be43d321e642b636f5b3906a62fb97d1761d1468 Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Tue, 21 Jul 2026 12:29:17 +0800 Subject: [PATCH 04/10] test(cpufreq): update voltage-envelope test to the raised 1.0V ceiling --- drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs index 79c20b9b17..b9605a6036 100644 --- a/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs +++ b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs @@ -696,13 +696,17 @@ mod tests { } #[test] - fn envelope_is_down_only_675_to_800() { - assert!(in_envelope(675_000)); - assert!(in_envelope(800_000)); + fn envelope_is_675k_to_1000k_inclusive() { + // Safe envelope is [VDD_FLOOR_UV, VDD_CEIL_UV] = [675 mV, 1.0 V]. The + // ceiling was raised from the boot 800 mV to the calibrated 1.0 V + // over-volted safe max needed for the higher A76/A55 OPPs; both bounds + // are inclusive. + assert!(in_envelope(VDD_FLOOR_UV)); // 675 mV floor, inclusive assert!(in_envelope(725_000)); - assert!(!in_envelope(674_999)); // below OPP floor - assert!(!in_envelope(800_001)); // above boot voltage - assert!(!in_envelope(1_000_000)); + assert!(in_envelope(800_000)); + assert!(in_envelope(VDD_CEIL_UV)); // 1.0 V ceiling, inclusive + assert!(!in_envelope(VDD_FLOOR_UV - 1)); // below floor + assert!(!in_envelope(VDD_CEIL_UV + 1)); // above ceiling } #[test] From ce32bc57ccc3a31b3e4a40878e5c54e71206c7ad Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Thu, 23 Jul 2026 11:54:43 +0800 Subject: [PATCH 05/10] fix(cpufreq): PMIC lock keeps IRQs on + host tests for OPP/governor state machine Blocker 1 (IRQ latency): the RK3588 PMIC I2C/SPI drivers held a `SpinNoIrq` across the entire slow, busy-waiting hardware transaction, disabling local IRQs for the whole PMIC poll. Switch both `pmic_i2c` and `pmic_spi` to `SpinNoPreempt` so IRQs stay enabled during the poll while a task switch still cannot interleave mid-transaction. Sound because the PMIC lock is never taken from an interrupt handler: every caller runs in the boot probe (`align_rail_voltages_to_opp`) or the sleepable `cpufreq` governor/calibration task, so an IRQ arriving mid-transaction can never re-enter and self-deadlock. Blocker 2 (untested state-machine fixes): extract the pure, behavior-preserving decision logic and add host `#[cfg(test)]` unit tests. - `apply_opp` now runs its two levers through pure `apply_step_order` + `run_apply_steps`: ordering (up = voltage-then-clock, down = clock-then-voltage) and the transactional stop-at-first-failure / commit-only-on-both-success rule that gates the governor's `IDX` advance. Hardware writes and their exact no-undervolt logs are unchanged. - `governor_poll`'s up/down/hold/prime decision is extracted to pure `next_opp_idx(core_pcts, cur, ladder_len, priming)`; the stateful per-core LAST_BUSY loop and logging are unchanged. - Tests cover: apply ordering + all partial-failure paths (no phantom OPP), governor fast-attack / one-step decay / hold / floor / priming, and the OPP tables staying within each rail's PMIC voltage envelope and boot-safe ring ceiling. --- drivers/ax-driver/src/soc/rockchip/cpufreq.rs | 390 +++++++++++++++--- .../ax-driver/src/soc/rockchip/pmic_i2c.rs | 11 +- .../ax-driver/src/soc/rockchip/pmic_spi.rs | 13 +- 3 files changed, 348 insertions(+), 66 deletions(-) diff --git a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs index 7b0a5285e5..0077c370d4 100644 --- a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs +++ b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs @@ -506,6 +506,43 @@ impl Cluster { } } +/// One of the two hardware levers an OPP transition programs. Naming them makes +/// the safety-critical apply *ordering* assertable by a host test with no +/// hardware (see [`apply_step_order`]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum ApplyStep { + /// Program the rail voltage (PMIC I2C/SPI write). + Voltage, + /// Program the SCMI PVTPLL ring (clock rate). + Clock, +} + +/// Ordering policy for [`apply_opp`], pure so a host test can pin it: to keep the +/// voltage-coupled clock from ever overshooting its rail, an UPSHIFT raises the +/// voltage before the clock (clock follows up), and a DOWNSHIFT lowers the clock +/// before the voltage (clock follows down). +const fn apply_step_order(going_up: bool) -> [ApplyStep; 2] { + if going_up { + [ApplyStep::Voltage, ApplyStep::Clock] + } else { + [ApplyStep::Clock, ApplyStep::Voltage] + } +} + +/// Transactional core of [`apply_opp`], pure so a host test can drive every +/// success/failure combination. Runs the ordered steps, STOPS at the first failed +/// step (so the second lever is never moved after the first failed), and returns +/// `true` only when BOTH steps are confirmed. `run(step)` performs one hardware +/// step and reports whether it was confirmed. +fn run_apply_steps(order: [ApplyStep; 2], mut run: impl FnMut(ApplyStep) -> bool) -> bool { + for step in order { + if !run(step) { + return false; + } + } + true +} + /// Apply an OPP to a domain as a matched (voltage, frequency) pair, ordered so /// the voltage-coupled clock never overshoots its rail: /// - going UP: raise voltage first, then the SCMI ring (clock follows up); @@ -516,7 +553,9 @@ impl Cluster { /// window this ordering exists to prevent), and returns `true` only when BOTH /// steps are confirmed. The caller ([`governor_poll`]) must commit its software /// OPP index only on `true`; on `false` it must keep tracking the last -/// confirmed index so the next poll retries from there. +/// confirmed index so the next poll retries from there. The ordering + +/// stop-on-failure logic is the pure, host-tested [`apply_step_order`] + +/// [`run_apply_steps`]; only the hardware writes and their logs live here. /// /// No-undervolt argument (both failure points, each direction): /// - UPSHIFT, voltage step fails: the ring set is skipped entirely, so @@ -536,53 +575,57 @@ impl Cluster { fn apply_opp(cluster: Cluster, opp: Opp, going_up: bool) -> bool { let phandle = Phandle::from(0u32); let hz = opp.ring_khz as u64 * 1_000; - if going_up { - if !cluster.set_voltage(opp.uv) { - warn!( - "cpufreq: {} upshift to {} mV failed; leaving clock id {} unchanged (no \ - undervolt: old freq stays paired with old voltage)", - cluster.name(), - opp.uv / 1_000, - cluster.clock_id() - ); - return false; - } - if scmi::set_clock_rate(phandle, cluster.clock_id(), hz).is_none() { - warn!( - "cpufreq: {} voltage already raised to {} mV but SCMI rejected clock id {} -> {} \ - Hz; not committing this OPP (safe: over-volted for the old, lower clock, never \ - under-volted)", - cluster.name(), - opp.uv / 1_000, - cluster.clock_id(), - hz - ); - return false; - } - } else { - if scmi::set_clock_rate(phandle, cluster.clock_id(), hz).is_none() { - warn!( - "cpufreq: {} downshift: SCMI rejected clock id {} -> {} Hz; leaving voltage \ - unchanged (no undervolt: old freq stays paired with old voltage)", - cluster.name(), - cluster.clock_id(), - hz - ); - return false; + run_apply_steps(apply_step_order(going_up), |step| match step { + ApplyStep::Voltage => { + if cluster.set_voltage(opp.uv) { + return true; + } + if going_up { + warn!( + "cpufreq: {} upshift to {} mV failed; leaving clock id {} unchanged (no \ + undervolt: old freq stays paired with old voltage)", + cluster.name(), + opp.uv / 1_000, + cluster.clock_id() + ); + } else { + warn!( + "cpufreq: {} clock already lowered to {} Hz but voltage write to {} mV \ + failed; not committing this OPP (safe: over-volted for the new, lower clock, \ + never under-volted)", + cluster.name(), + hz, + opp.uv / 1_000 + ); + } + false } - if !cluster.set_voltage(opp.uv) { - warn!( - "cpufreq: {} clock already lowered to {} Hz but voltage write to {} mV failed; \ - not committing this OPP (safe: over-volted for the new, lower clock, never \ - under-volted)", - cluster.name(), - hz, - opp.uv / 1_000 - ); - return false; + ApplyStep::Clock => { + if scmi::set_clock_rate(phandle, cluster.clock_id(), hz).is_some() { + return true; + } + if going_up { + warn!( + "cpufreq: {} voltage already raised to {} mV but SCMI rejected clock id {} -> \ + {} Hz; not committing this OPP (safe: over-volted for the old, lower clock, \ + never under-volted)", + cluster.name(), + opp.uv / 1_000, + cluster.clock_id(), + hz + ); + } else { + warn!( + "cpufreq: {} downshift: SCMI rejected clock id {} -> {} Hz; leaving voltage \ + unchanged (no undervolt: old freq stays paired with old voltage)", + cluster.name(), + cluster.clock_id(), + hz + ); + } + false } - } - true + }) } /// Period, in ms, the kernel governor task sleeps between [`governor_poll`]s. @@ -626,6 +669,33 @@ pub fn governor_period_ms() -> u64 { GOV_PERIOD_MS } +/// Pure ondemand policy, split out of [`governor_poll`] so the up/down/hold/prime +/// decision is host-testable with no hardware or tick counters. Given each core's +/// busy% over the last window (each already clamped to `0..=100`), the cluster's +/// current OPP index `cur`, the ladder length, and whether this is the priming +/// poll, return the next OPP index: +/// * priming poll, an empty cluster, or an empty ladder -> hold (no change); +/// * any core at/above [`UP_THRESHOLD_PCT`] -> jump straight to the top OPP +/// (ondemand fast-attack; a cluster shares one clock, so its busiest core +/// drives it — averaging would bury a single saturated thread); +/// * every core below [`DOWN_THRESHOLD_PCT`] -> shed exactly one step (never +/// below index 0); +/// * otherwise hold. +fn next_opp_idx(core_pcts: &[u64], cur: usize, ladder_len: usize, priming: bool) -> usize { + if priming || core_pcts.is_empty() || ladder_len == 0 { + return cur; + } + let any_core_high = core_pcts.iter().any(|&p| p >= UP_THRESHOLD_PCT); + let all_cores_low = core_pcts.iter().all(|&p| p < DOWN_THRESHOLD_PCT); + if any_core_high { + ladder_len - 1 + } else if all_cores_low && cur > 0 { + cur - 1 + } else { + cur + } +} + /// One ondemand iteration, called periodically by the kernel governor task with /// a fresh snapshot of every CPU's cumulative busy-tick counter /// (`ax_task::cpu_busy_ticks`). Scores each core's busy% over the last window and @@ -668,39 +738,37 @@ pub fn governor_poll(busy: &[u64]) { // thread on the 2-core A76 pair only reads 50%, below the up-threshold, // so the cluster never boosted. Each CPU belongs to exactly one cluster, // so every LAST_BUSY entry is refreshed exactly once per poll. - let mut any_core_high = false; // some core wants the top OPP - let mut all_cores_low = true; // every core is near-idle → shed one step let mut peak_pct = 0u64; // busiest core, for the log line - let mut n = 0u64; + // RK3588 clusters have at most 4 cores (A55 cpu0-3; the big pairs have 2). + let mut core_pcts = [0u64; 4]; + let mut n = 0usize; for cpu in cluster.cpus() { let now = busy.get(cpu).copied().unwrap_or(0); let last = LAST_BUSY[cpu].swap(now, Ordering::Relaxed); // Per-core busy% = busy_ticks / window_ticks (one core), clamped. let pct = ((now.saturating_sub(last) * 100) / WINDOW_TICKS).min(100); - if pct >= UP_THRESHOLD_PCT { - any_core_high = true; - } - if pct >= DOWN_THRESHOLD_PCT { - all_cores_low = false; - } if pct > peak_pct { peak_pct = pct; } + if let Some(slot) = core_pcts.get_mut(n) { + *slot = pct; + } n += 1; } - if n == 0 || priming { + if n == 0 { continue; } let opps = cluster.opps(); let cur = IDX[ci].load(Ordering::Relaxed); - let new = if any_core_high { - opps.len() - 1 // fast attack: jump straight to the top OPP - } else if all_cores_low && cur > 0 { - cur - 1 // slow decay: shed one step only when the whole cluster is idle - } else { - cur - }; + // Pure up/down/hold/prime decision (host-tested); the priming poll only + // seeds the baseline above and holds here. + let new = next_opp_idx( + &core_pcts[..n.min(core_pcts.len())], + cur, + opps.len(), + priming, + ); if new != cur { // Only commit IDX (the software record of the last CONFIRMED OPP) @@ -926,3 +994,199 @@ pub fn calibrate_cluster(cluster_idx: usize, intended_cpu: usize) { restore_khz / 1_000 ); } + +#[cfg(test)] +mod tests { + use super::*; + + // ----------------------------------------------------------------------- + // OPP transactional apply: ordering + commit-only-on-full-success + // + // These pin the two prior state-machine fixes: `apply_opp` orders the levers + // so the voltage-coupled clock never overshoots, and commits (returns `true`, + // which is what lets `governor_poll` advance `IDX`) ONLY when both the voltage + // and SCMI steps succeed. They exercise the exact pure helpers `apply_opp` + // uses, so a partial-failure regression (advancing past a step that failed) + // would fail here. + // ----------------------------------------------------------------------- + + /// Run `run_apply_steps` with injected per-step outcomes (keyed by execution + /// order), recording which steps actually ran — exactly how `apply_opp` drives + /// the two levers, minus the hardware. + fn run_recording(order: [ApplyStep; 2], outcomes: [bool; 2]) -> (bool, [Option; 2]) { + let mut ran: [Option; 2] = [None, None]; + let mut i = 0usize; + let committed = run_apply_steps(order, |step| { + ran[i] = Some(step); + let ok = outcomes[i]; + i += 1; + ok + }); + (committed, ran) + } + + #[test] + fn apply_step_order_up_is_voltage_then_clock_down_is_clock_then_voltage() { + assert_eq!( + apply_step_order(true), + [ApplyStep::Voltage, ApplyStep::Clock] + ); + assert_eq!( + apply_step_order(false), + [ApplyStep::Clock, ApplyStep::Voltage] + ); + } + + #[test] + fn apply_commits_only_when_both_steps_succeed() { + let (committed, ran) = run_recording(apply_step_order(true), [true, true]); + assert!(committed, "both steps confirmed must commit the OPP"); + assert_eq!(ran, [Some(ApplyStep::Voltage), Some(ApplyStep::Clock)]); + } + + #[test] + fn apply_first_step_failure_skips_second_and_does_not_commit() { + // UPSHIFT with the voltage (first) step failing: the SCMI ring must NOT be + // touched, and the OPP must not commit. + let (committed, ran) = run_recording(apply_step_order(true), [false, true]); + assert!(!committed, "a failed first step must not commit the OPP"); + assert_eq!( + ran, + [Some(ApplyStep::Voltage), None], + "the second lever must be skipped after the first step fails" + ); + } + + #[test] + fn apply_second_step_failure_does_not_commit() { + // DOWNSHIFT with the voltage (second) step failing after the ring lowered: + // both steps ran, but the OPP must not commit (hardware is left over-volted + // for the new lower clock — safe, never under-volted). + let (committed, ran) = run_recording(apply_step_order(false), [true, false]); + assert!(!committed, "a failed second step must not commit the OPP"); + assert_eq!(ran, [Some(ApplyStep::Clock), Some(ApplyStep::Voltage)]); + } + + // ----------------------------------------------------------------------- + // Ondemand governor policy (pure): up / down / hold / prime / floor + // ----------------------------------------------------------------------- + + #[test] + fn governor_saturated_core_jumps_to_top_opp() { + // A single saturated core in a 2-core big pair drives the whole cluster to + // its top OPP (fast attack; the up-threshold is inclusive). + let top = A76_OPPS.len() - 1; + assert_eq!( + next_opp_idx(&[100, 0], BOOT_OPP_IDX, A76_OPPS.len(), false), + top + ); + assert_eq!( + next_opp_idx(&[UP_THRESHOLD_PCT, 0], 0, A76_OPPS.len(), false), + top + ); + } + + #[test] + fn governor_all_idle_sheds_one_step() { + assert_eq!( + next_opp_idx(&[0, 0], BOOT_OPP_IDX, A76_OPPS.len(), false), + BOOT_OPP_IDX - 1 + ); + // Every A55 core just under the down-threshold sheds exactly one step. + assert_eq!( + next_opp_idx(&[DOWN_THRESHOLD_PCT - 1; 4], 3, A55_OPPS.len(), false), + 2 + ); + } + + #[test] + fn governor_does_not_shed_below_bottom_opp() { + assert_eq!(next_opp_idx(&[0, 0], 0, A76_OPPS.len(), false), 0); + } + + #[test] + fn governor_holds_on_moderate_load() { + // No core saturated and not every core idle: hold. + let mid = (DOWN_THRESHOLD_PCT + UP_THRESHOLD_PCT) / 2; + assert_eq!( + next_opp_idx(&[mid, mid], BOOT_OPP_IDX, A76_OPPS.len(), false), + BOOT_OPP_IDX + ); + // One busy-but-not-saturated core keeps the cluster put (down-threshold is + // exclusive at the top, up-threshold not yet reached). + assert_eq!( + next_opp_idx( + &[UP_THRESHOLD_PCT - 1, 0], + BOOT_OPP_IDX, + A76_OPPS.len(), + false + ), + BOOT_OPP_IDX + ); + } + + #[test] + fn governor_priming_and_degenerate_inputs_make_no_change() { + // The first (priming) poll only seeds the busy baseline; even a saturated + // reading must not move the OPP that window. + assert_eq!( + next_opp_idx(&[100, 100], BOOT_OPP_IDX, A76_OPPS.len(), true), + BOOT_OPP_IDX + ); + // An empty cluster or an empty ladder also holds. + assert_eq!( + next_opp_idx(&[], BOOT_OPP_IDX, A76_OPPS.len(), false), + BOOT_OPP_IDX + ); + assert_eq!(next_opp_idx(&[100], BOOT_OPP_IDX, 0, false), BOOT_OPP_IDX); + } + + // ----------------------------------------------------------------------- + // OPP-table invariants + // ----------------------------------------------------------------------- + + #[test] + fn opp_table_voltages_within_pmic_envelope() { + // Every ladder rung must be a voltage its rail's PMIC layer will actually + // accept, or the governor's transition would silently fail. The A76 rails + // (RK8602/03 over I2C) accept [VDD_FLOOR_UV, VDD_CEIL_UV] = [675_000, + // 1_000_000] uV (see pmic_i2c); the A55 rail (RK806 force-write over SPI) + // accepts [675_000, 950_000] uV (see pmic_spi::force_write_dcdc2). Both + // bounds are inclusive. (This complements pmic_i2c's own envelope test.) + for opp in A76_OPPS { + assert!( + (675_000..=1_000_000).contains(&opp.uv), + "A76 OPP {} mV outside the I2C PMIC envelope", + opp.uv / 1_000 + ); + } + for opp in A55_OPPS { + assert!( + (675_000..=950_000).contains(&opp.uv), + "A55 OPP {} mV outside the SPI PMIC envelope", + opp.uv / 1_000 + ); + } + } + + #[test] + fn opp_table_rings_within_boot_safe_ceiling() { + // The ladders never ring a cluster above its boot-safe ceiling: the top + // rungs hold the boot ring and raise only voltage. Guards against a future + // rung that would ring past the ceiling the probe verified. + for opp in A76_OPPS { + assert!( + opp.ring_khz as u64 * 1_000 <= A76_MAX_HZ, + "A76 ring {} kHz above the boot-safe ceiling", + opp.ring_khz + ); + } + for opp in A55_OPPS { + assert!( + opp.ring_khz as u64 * 1_000 <= A55_MAX_HZ, + "A55 ring {} kHz above the boot-safe ceiling", + opp.ring_khz + ); + } + } +} diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs index b9605a6036..ef832bc869 100644 --- a/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs +++ b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs @@ -56,7 +56,16 @@ use core::time::Duration; -use ax_kspin::SpinNoIrq as Mutex; +// PMIC access is slow (register reads/writes that busy-wait on I2C hardware), so +// the lock is held across the whole transaction. Use `SpinNoPreempt`, not +// `SpinNoIrq`: it keeps local IRQs ENABLED during the poll (so IRQ latency is not +// held hostage to a millisecond PMIC transaction) while still disabling preemption +// so no task switch can interleave mid-transaction and corrupt the register +// sequence. This is sound because the lock is NEVER taken from an interrupt +// handler — every caller (`init`/`get_uv`/`set_uv*`) runs in the boot probe or the +// sleepable `cpufreq` governor task, so an IRQ arriving mid-transaction can never +// re-enter and self-deadlock. +use ax_kspin::SpinNoPreempt as Mutex; use log::{info, warn}; use mmio_api::{MmioAddr, MmioRaw}; use rdif_pinctrl::PinctrlDevice; diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs b/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs index 5db319e2ad..60605b9ff1 100644 --- a/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs +++ b/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs @@ -83,7 +83,16 @@ use core::time::Duration; -use ax_kspin::SpinNoIrq as Mutex; +// PMIC access is slow (SPI register reads/writes that busy-wait on hardware), so +// the lock is held across the whole transaction. Use `SpinNoPreempt`, not +// `SpinNoIrq`: it keeps local IRQs ENABLED during the poll (so IRQ latency is not +// held hostage to a millisecond SPI transaction) while still disabling preemption +// so no task switch can interleave mid-transaction and corrupt the register +// sequence. This is sound because the lock is NEVER taken from an interrupt +// handler — every caller (`init`/`get_uv`/`set_uv*`/`force_write_dcdc2`) runs in +// the boot probe or the sleepable `cpufreq` governor task, so an IRQ arriving +// mid-transaction can never re-enter and self-deadlock. +use ax_kspin::SpinNoPreempt as Mutex; use log::{info, warn}; use rdif_pinctrl::PinctrlDevice; @@ -257,7 +266,7 @@ struct Rk806Spi { base: *mut u8, } -// The controller is reached only through `PMIC` (a `SpinNoIrq` mutex), which +// The controller is reached only through `PMIC` (a `SpinNoPreempt` mutex), which // serializes all access; the MMIO mapping is stable for the kernel lifetime. unsafe impl Send for Rk806Spi {} From 555b6ba1ad72f0794c1b9f2efa03f5d06d543913 Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Thu, 23 Jul 2026 14:47:59 +0800 Subject: [PATCH 06/10] =?UTF-8?q?fix(cpufreq):=20A55=20ring-only=20DVFS=20?= =?UTF-8?q?=E2=80=94=20drop=20unconfirmable=20dynamic=20voltage=20writes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RK806 DCDC2 rail (A55) accepts writes (rail-alignment freq drops confirm) but its SPI read is a hardware scope-wall — MISO never feeds the shift register, so a dynamic voltage write cannot be read back to confirm the rail reached target. Per review, scaling the A55 voltage on such an unconfirmable write risks committing an OPP whose rail never arrived (undervolt). Cluster-split the readiness: the A76 rails read back over I2C and keep the full voltage lever; the A55 cluster forgoes it. A55 is pinned to the boot-confirmed 675 mV rail and scales only its SCMI PVTPLL ring (408/816/1008 MHz). 675 mV over-volts every ring <= 1008 (board-measured to deliver 1008 exactly), so no A55 rung can undervolt regardless of what the unconfirmable SPI write did. - Truncate A55_OPPS to the three 675 mV ring-only rungs (drop the voltage-scaled 1212-1523 MHz rungs). The canonical A55 voltages (mainline rk3588-opp.dtsi cluster0: 1200@712.5, 1416@762.5, 1608@850, 1800@950; ceiling 950 mV) are Linux-rated but still need a trusted RK806 write-back before we drive them. - Add Cluster::voltage_managed(); apply_opp treats the voltage step as a confirmed no-op for ring-only clusters, so the ordered/transactional apply is unchanged but issues no dynamic A55 SPI write. - Lock the invariant with a host test (a55_ladder_is_ring_only_on_the_boot_rail). 33 host tests pass; fmt + clippy clean. --- drivers/ax-driver/src/soc/rockchip/cpufreq.rs | 101 +++++++++++++----- 1 file changed, 75 insertions(+), 26 deletions(-) diff --git a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs index 0077c370d4..344c7c6694 100644 --- a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs +++ b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs @@ -402,9 +402,20 @@ const A76_OPPS: &[Opp] = &[ }, ]; -/// A55 (little) OPP ladder, low→high, same hybrid rationale: ring-scaled below the -/// 675 mV point, then ring 1008 with rising voltage. Top rung 1523 MHz @ 950 mV -/// (the RK806 force-write ceiling), the sweep's safe maximum for the little cluster. +/// A55 (little) OPP ladder, low→high. Unlike the A76 ladder this is **ring-only**: +/// every rung sits on the boot-confirmed 675 mV rail and only the SCMI PVTPLL ring +/// moves. The A55 rail is RK806 DCDC2 over SPI2, whose *read* path is a hardware +/// scope-wall — writes reach the chip (proven by the rail-alignment freq drop) but +/// MISO never feeds the shift register, so a write cannot be read back to confirm +/// the rail actually reached target. Scaling the A55 voltage on such an unconfirmable +/// write could commit an OPP whose rail never arrived; rather than risk that, the +/// little cluster forgoes the voltage lever and stays on 675 mV, which board +/// measurement proves delivers the full 1008 MHz ring exactly. 675 mV over-volts +/// every ring at or below 1008, so no A55 rung can undervolt regardless of the PMIC +/// (see [`Cluster::voltage_managed`]). The big clusters keep the voltage lever +/// because their RK8602/RK8603 rails read back over I2C. Restoring the higher +/// voltage-scaled A55 rungs (1212–1523 MHz) is gated on a trusted RK806 write-back +/// (an oscilloscope-level MISO fix, or a delivered-frequency oracle confirmation). const A55_OPPS: &[Opp] = &[ Opp { ring_khz: 408_000, @@ -421,26 +432,6 @@ const A55_OPPS: &[Opp] = &[ uv: 675_000, mhz: 1021, }, - Opp { - ring_khz: 1_008_000, - uv: 762_500, - mhz: 1212, - }, - Opp { - ring_khz: 1_008_000, - uv: 800_000, - mhz: 1285, - }, - Opp { - ring_khz: 1_008_000, - uv: 850_000, - mhz: 1372, - }, - Opp { - ring_khz: 1_008_000, - uv: 950_000, - mhz: 1523, - }, ]; /// Index into each ladder of the boot OPP the voltage lever leaves the cluster on: @@ -496,6 +487,10 @@ impl Cluster { /// regulator; A55 uses the bounded force-write (its RK806 read is a /// scope-wall, but writes reach the chip — proven by the rail-alignment freq drop). /// Both PMIC helpers clamp to their rail's safe envelope internally. + /// + /// Only ever called for a [`voltage_managed`](Self::voltage_managed) domain — + /// the A55 arm remains for the one-time boot alignment path but the governor + /// never drives it, so no dynamic (unconfirmable) A55 SPI write is issued. fn set_voltage(self, uv: u32) -> bool { use super::{pmic_i2c, pmic_spi}; match self { @@ -504,6 +499,21 @@ impl Cluster { Cluster::Big1 => pmic_i2c::set_uv(pmic_i2c::RK8603_BIG1_ADDR, uv), } } + + /// Whether the governor may scale this domain's rail voltage dynamically. + /// + /// The A76 rails (RK8602/RK8603 over I2C) read back, so each write is confirmed + /// before the clock is allowed to follow it up — they get the full voltage lever. + /// The A55 rail (RK806 DCDC2 over SPI2) cannot be read back (a MISO hardware + /// scope-wall), so its voltage is programmed exactly once at init — the down-only, + /// boot-safe 675 mV alignment — and never scaled dynamically. The A55 ladder is + /// therefore ring-only and every A55 rung stays over-volted (675 mV supports the + /// whole ≤1008 MHz ring), which is undervolt-safe no matter what the unconfirmable + /// SPI write actually did. This per-cluster split keeps the readback-verified A76 + /// DVFS while removing every dynamic A55 voltage write. + fn voltage_managed(self) -> bool { + !matches!(self, Cluster::A55) + } } /// One of the two hardware levers an OPP transition programs. Naming them makes @@ -577,6 +587,16 @@ fn apply_opp(cluster: Cluster, opp: Opp, going_up: bool) -> bool { let hz = opp.ring_khz as u64 * 1_000; run_apply_steps(apply_step_order(going_up), |step| match step { ApplyStep::Voltage => { + // Ring-only clusters (A55) never scale voltage dynamically: their rail + // was pinned once at init to the boot-safe 675 mV row (down-only, and + // over-volted for every ring ≤ 1008) and cannot be read back to confirm + // a dynamic write. Treat the voltage step as a confirmed no-op so the + // ordered/transactional apply logic is unchanged while no unconfirmable + // SPI write is ever issued and the clock (already within the fixed rail's + // envelope) can safely follow. See [`Cluster::voltage_managed`]. + if !cluster.voltage_managed() { + return true; + } if cluster.set_voltage(opp.uv) { return true; } @@ -1092,10 +1112,16 @@ mod tests { next_opp_idx(&[0, 0], BOOT_OPP_IDX, A76_OPPS.len(), false), BOOT_OPP_IDX - 1 ); - // Every A55 core just under the down-threshold sheds exactly one step. + // Every A55 core just under the down-threshold sheds exactly one step + // (from the top of the ring-only A55 ladder down one rung). assert_eq!( - next_opp_idx(&[DOWN_THRESHOLD_PCT - 1; 4], 3, A55_OPPS.len(), false), - 2 + next_opp_idx( + &[DOWN_THRESHOLD_PCT - 1; 4], + A55_OPPS.len() - 1, + A55_OPPS.len(), + false + ), + A55_OPPS.len() - 2 ); } @@ -1169,6 +1195,29 @@ mod tests { } } + #[test] + fn a55_ladder_is_ring_only_on_the_boot_rail() { + // The A55 RK806 rail cannot be read back (MISO scope-wall), so the little + // cluster forgoes the voltage lever: every A55 rung must sit on the single + // boot-confirmed 675 mV rail (see `Cluster::voltage_managed`). This locks in + // the cluster split — a future rung that raised the A55 voltage on an + // unconfirmable SPI write would fail here. + assert!( + !Cluster::A55.voltage_managed(), + "A55 must stay voltage-unmanaged (ring-only) until its rail reads back" + ); + for opp in A55_OPPS { + assert_eq!( + opp.uv, 675_000, + "A55 rung {} MHz must stay on the 675 mV boot rail (ring-only)", + opp.mhz + ); + } + // The A76 clusters keep the full voltage lever. + assert!(Cluster::Big0.voltage_managed()); + assert!(Cluster::Big1.voltage_managed()); + } + #[test] fn opp_table_rings_within_boot_safe_ceiling() { // The ladders never ring a cluster above its boot-safe ceiling: the top From 414672ac047392c9a721a8d0e7ffef6c494db92f Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Thu, 23 Jul 2026 17:13:44 +0800 Subject: [PATCH 07/10] fix(cpufreq): read back the SCMI clock rate before committing an OPP transition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CLOCK_RATE_SET ack from SCMI is not proof the PVTPLL ring actually switched. governor_poll's apply_opp treated `set_clock_rate(...).is_some()` as a confirmed switch, so on a downshift (clock-then-voltage) it could go on to lower the rail while the clock was still high — the freq-high/voltage-low window the apply ordering exists to prevent. Confirm the clock like the boot `set_and_verify` already does: after the SET, read the rate back with `scmi::clock_rate` and require it to equal the request; only then is the clock step successful. Because the step is transactional and stops on failure, a downshift now reaches its voltage-lower step only once the clock is verified already lowered, and an unconfirmed clock leaves the rail untouched (over-volted for the old clock, never under-volted). Updated the apply_opp no-undervolt doc to state each step is read-back-confirmed. 33 host tests pass; fmt + clippy -D warnings clean. --- drivers/ax-driver/src/soc/rockchip/cpufreq.rs | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs index 344c7c6694..f9f5745deb 100644 --- a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs +++ b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs @@ -567,6 +567,12 @@ fn run_apply_steps(order: [ApplyStep; 2], mut run: impl FnMut(ApplyStep) -> bool /// stop-on-failure logic is the pure, host-tested [`apply_step_order`] + /// [`run_apply_steps`]; only the hardware writes and their logs live here. /// +/// Each step is CONFIRMED, not just requested: the clock step reads the SCMI rate +/// back and requires it to equal the request (a SET ack alone is not proof the ring +/// switched), and the A76 voltage step reads back over I2C. So "ring set succeeds" +/// below means the ring is read-back-confirmed at the new rate — a downshift only +/// reaches its voltage-lower step once the clock is verified already lowered. +/// /// No-undervolt argument (both failure points, each direction): /// - UPSHIFT, voltage step fails: the ring set is skipped entirely, so /// nothing changed — old freq + old voltage, still a valid, previously @@ -621,26 +627,38 @@ fn apply_opp(cluster: Cluster, opp: Opp, going_up: bool) -> bool { false } ApplyStep::Clock => { - if scmi::set_clock_rate(phandle, cluster.clock_id(), hz).is_some() { + let cid = cluster.clock_id(); + // A SCMI CLOCK_RATE_SET ack is NOT proof the PVTPLL ring actually + // switched — read the rate back and require it to equal the request + // before treating the clock step as confirmed (same read-back the boot + // `set_and_verify` does). This is what makes a DOWNSHIFT safe: the + // voltage step that follows only runs once the clock is CONFIRMED at its + // new, lower rate, so we can never lower the rail under a still-high + // clock (the high-freq/low-voltage window the ordering exists to prevent). + let set_ok = scmi::set_clock_rate(phandle, cid, hz).is_some(); + let applied = if set_ok { + scmi::clock_rate(phandle, cid) + } else { + None + }; + if set_ok && applied == Some(hz) { return true; } if going_up { warn!( - "cpufreq: {} voltage already raised to {} mV but SCMI rejected clock id {} -> \ - {} Hz; not committing this OPP (safe: over-volted for the old, lower clock, \ - never under-volted)", - cluster.name(), - opp.uv / 1_000, - cluster.clock_id(), - hz + "cpufreq: {} upshift: SCMI clock id {cid} not confirmed at {hz} Hz \ + (set_ok={set_ok}, read_back={applied:?}); not committing this OPP (safe: \ + voltage already raised, over-volted for the old, lower clock, never \ + under-volted)", + cluster.name() ); } else { warn!( - "cpufreq: {} downshift: SCMI rejected clock id {} -> {} Hz; leaving voltage \ - unchanged (no undervolt: old freq stays paired with old voltage)", - cluster.name(), - cluster.clock_id(), - hz + "cpufreq: {} downshift: SCMI clock id {cid} not confirmed at {hz} Hz \ + (set_ok={set_ok}, read_back={applied:?}); leaving voltage unchanged (no \ + undervolt: clock not confirmed lowered, old freq stays paired with old \ + voltage)", + cluster.name() ); } false From 406c1ee45e0af276e10514cd701d47d486459fae Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Thu, 23 Jul 2026 20:22:04 +0800 Subject: [PATCH 08/10] feat(cpufreq): wire rk3588-cpufreq into the Orange Pi board build + document it Per review, the DVFS feature was never enabled in any board build/test/doc entry so CI only exercised the feature-off stub path. Enable it in the Orange Pi 5 Plus board config alongside the other default-on RK3588 hardware features (rknpu, rk3588-pwm), so the existing `Test starry self-hosted board orangepi-5-plus` entry builds and runs the governor on real hardware. It is fail-safe: if a PMIC bus does not come up, GOV_READY stays false and each cluster is left on its boot OPP. Add docs/rk3588-cpufreq.md with the reproducible enable command (`cargo xtask starry test board --board orangepi-5-plus`) and the observable governor/OPP verification: the boot arming log, per-cluster `gov:` OPP transitions, and cpuprobe `mhz_pmc` under load. --- docs/rk3588-cpufreq.md | 55 +++++++++++++++++++ .../configs/board/orangepi-5-plus.toml | 1 + 2 files changed, 56 insertions(+) create mode 100644 docs/rk3588-cpufreq.md diff --git a/docs/rk3588-cpufreq.md b/docs/rk3588-cpufreq.md new file mode 100644 index 0000000000..ed196d41e8 --- /dev/null +++ b/docs/rk3588-cpufreq.md @@ -0,0 +1,55 @@ +# RK3588 CPU DVFS (cpufreq) + +Feature-gated ondemand CPU frequency/voltage scaling for the Orange Pi 5 Plus +(RK3588). It is off by default in the generic and QEMU builds — the +`ax-driver/rk3588-cpufreq` feature compiles the driver to no-ops there — and is +enabled on the Orange Pi board build. + +## What it does + +The RK3588 CPU clock is a voltage-coupled PVTPLL. An **ondemand** governor scales +each of the three CPU clusters (A55 little `cpu0-3`, two A76 big pairs `cpu4-5` / +`cpu6-7`) between OPPs by pairing an SCMI PVTPLL ring target with a PMIC rail +voltage: + +- **A76** uses the full voltage lever — its RK8602/RK8603 rails read back over I2C, + so every voltage write is confirmed and the SCMI clock rate is read back before an + OPP is committed. +- **A55** is **ring-only**: its RK806 rail cannot be read back (a MISO hardware + limit), so it stays on the boot-confirmed 675 mV rail and scales only its SCMI + ring. 675 mV over-volts every A55 ring ≤ 1008 MHz, so it can never undervolt. + +It is **fail-safe**: if either PMIC bus does not come up at boot, `GOV_READY` stays +`false` and every cluster is left on its boot OPP (no scaling). + +## Enable and build + +The feature is wired into the Orange Pi 5 Plus board config +(`os/StarryOS/configs/board/orangepi-5-plus.toml`), so the standard board +build/test entry enables it: + +```bash +cargo xtask starry test board --board orangepi-5-plus +``` + +To enable it in a custom build, add `"ax-driver/rk3588-cpufreq"` to that config's +`features` list. + +## Observable verification + +At boot (early init, before the console handoff) the driver logs the rail bring-up +and governor arming to the serial console: + +``` +cpufreq: A55 rail boot voltage = uV +cpufreq: A55 ->, A76 -> MHz +cpufreq: ondemand governor armed (both PMIC buses up) +``` + +Under load, per-cluster OPP transitions are logged +(`gov: A76b0 peak=% opp -> = MHz @ mV`), and the delivered +frequency can be read exactly with `cpuprobe`'s `mhz_pmc` (the PMU cycle counter is +enabled at boot): a busy cluster climbs toward its top OPP, an idle cluster sheds +back down one step at a time. The companion `apps/starry/sysbench-board` harness +(PR #1658) drives an all-core load and captures these numbers against a Linux +baseline. diff --git a/os/StarryOS/configs/board/orangepi-5-plus.toml b/os/StarryOS/configs/board/orangepi-5-plus.toml index 416648573f..31df15cdf5 100644 --- a/os/StarryOS/configs/board/orangepi-5-plus.toml +++ b/os/StarryOS/configs/board/orangepi-5-plus.toml @@ -10,6 +10,7 @@ features = [ "ax-driver/rockchip-ehci", "ax-driver/rockchip-sdhci", "ax-driver/rockchip-dwmmc", + "ax-driver/rk3588-cpufreq", "rknpu", "rk3588-pwm", ] From 861e6a11ee6fe832c217ce5321e7d64407b2c20f Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Sat, 25 Jul 2026 19:33:49 +0800 Subject: [PATCH 09/10] feat(cpufreq): enable rk3588-cpufreq in the board CI build config (not just the template) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The board CI job `Test starry self-hosted board orangepi-5-plus` (`cargo xtask starry test board --board orangepi-5-plus`) builds from `test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml`, NOT `os/StarryOS/configs/board/orangepi-5-plus.toml` (the general template the earlier commit edited). So the passing board CI never compiled or ran the governor. Add `ax-driver/rk3588-cpufreq` to the CI build wrapper so every board case now boots this kernel with the governor active: it arms and drives A55 DVFS at init (logs on the boot serial), and a governor-induced boot panic/hang trips the cases' `panic` fail-regex — giving the feature real board regression coverage. Kept at max_cpu_num=1 (the CI smoke build; only cpu0/A55 is loaded, A76 idles → lowest-risk on the shared board). Fail-safe as before: no PMIC bus up → GOV_READY stays false → boot OPP. Doc updated to point at the CI build config. --- docs/rk3588-cpufreq.md | 19 +++++++++++++------ .../build-aarch64-unknown-none-softfloat.toml | 1 + 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/rk3588-cpufreq.md b/docs/rk3588-cpufreq.md index ed196d41e8..92834225a1 100644 --- a/docs/rk3588-cpufreq.md +++ b/docs/rk3588-cpufreq.md @@ -24,13 +24,20 @@ It is **fail-safe**: if either PMIC bus does not come up at boot, `GOV_READY` st ## Enable and build -The feature is wired into the Orange Pi 5 Plus board config -(`os/StarryOS/configs/board/orangepi-5-plus.toml`), so the standard board -build/test entry enables it: +The feature is enabled in both Orange Pi 5 Plus build configs: -```bash -cargo xtask starry test board --board orangepi-5-plus -``` +- `test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml` + — the config the **board CI runner** actually builds, so the standard board + test entry compiles, boots and runs the governor on real hardware: + + ```bash + cargo xtask starry test board --board orangepi-5-plus + ``` + + Every case there boots this kernel; a governor-induced boot panic/hang trips the + cases' `panic` fail-regex, so the board CI regression-guards the feature. +- `os/StarryOS/configs/board/orangepi-5-plus.toml` — the general board build + template (for a full-board `max_cpu_num` build outside CI). To enable it in a custom build, add `"ax-driver/rk3588-cpufreq"` to that config's `features` list. diff --git a/test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml b/test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml index 6e76e2d328..77663747e1 100644 --- a/test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml +++ b/test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml @@ -10,6 +10,7 @@ features = [ "ax-driver/rockchip-ehci", "ax-driver/rockchip-sdhci", "ax-driver/rockchip-dwmmc", + "ax-driver/rk3588-cpufreq", "rknpu", "rk3588-pwm", ] From 5c2da73a67fb2750e6c80a741c6a463a9eb217ca Mon Sep 17 00:00:00 2001 From: JosephJoshua Date: Sun, 26 Jul 2026 12:47:57 +0800 Subject: [PATCH 10/10] fix(cpufreq): don't hold a preempt-disabling lock across PMIC hardware polls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both PMIC paths held a `SpinNoPreempt` guard across the whole chip transaction, so the ~100 ms I2C `wait_ipd` poll (and ~20 ms SPI transfers, longer if the bus is dead) ran with preemption disabled — the cpufreq governor task could stall task-switching on its CPU for the entire transaction, a scheduling-starvation risk when the bus misbehaves. Serialise the slow transactions with a non-preempt-disabling bus-ownership flag instead (`BUS_BUSY` + an RAII `BusGuard`): claim ownership with a non-blocking CAS, clone the controller handle out under a microsecond `SpinNoPreempt` window, then run the poll with preemption ENABLED. The `SpinNoPreempt` mutex now only ever guards storing/cloning the handle, never a poll. Concurrency protocol (made explicit, per review): the PMIC bus has one steady-state caller — the single governor task — and `init` runs once before it exists, so contention does not occur by construction. `BusGuard::claim` therefore takes ownership without blocking and *bails* to the caller's safe no-op if a transaction is somehow already in flight, rather than spinning. A poll preempted mid-transaction just holds ownership across the gap; the hardware transaction keeps running and resuming the status poll is idempotent. Behavior for the real single-caller path is unchanged (same register sequences/order). 33 host tests pass; fmt + clippy -D warnings clean. Board revalidation of the PMIC paths recommended before merge. --- .../ax-driver/src/soc/rockchip/pmic_i2c.rs | 117 ++++++++++++++---- .../ax-driver/src/soc/rockchip/pmic_spi.rs | 114 +++++++++++++---- 2 files changed, 182 insertions(+), 49 deletions(-) diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs index ef832bc869..2dcb88d3f1 100644 --- a/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs +++ b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs @@ -54,17 +54,24 @@ //! `di->vsel_min = 500000`, `di->vsel_step = 6250` → `V = 500000 + code·6250` //! µV, i.e. `0x1c → 675 mV`, `0x30 → 800 mV`. -use core::time::Duration; - -// PMIC access is slow (register reads/writes that busy-wait on I2C hardware), so -// the lock is held across the whole transaction. Use `SpinNoPreempt`, not -// `SpinNoIrq`: it keeps local IRQs ENABLED during the poll (so IRQ latency is not -// held hostage to a millisecond PMIC transaction) while still disabling preemption -// so no task switch can interleave mid-transaction and corrupt the register -// sequence. This is sound because the lock is NEVER taken from an interrupt -// handler — every caller (`init`/`get_uv`/`set_uv*`) runs in the boot probe or the -// sleepable `cpufreq` governor task, so an IRQ arriving mid-transaction can never -// re-enter and self-deadlock. +use core::{ + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +// PMIC access is slow: a register read/write busy-waits on the I2C hardware +// (`wait_ipd`, ~100 ms budget, longer if the bus is dead). Serialisation must NOT +// disable preemption across that poll, or the `cpufreq` governor task would stall +// task-switching on its CPU for the whole transaction (a scheduling-starvation +// risk when the bus misbehaves). So the slow chip transactions are guarded by a +// non-preempt-disabling bus-ownership flag (`BUS_BUSY` / [`BusGuard`]) and the +// `SpinNoPreempt` `Mutex` below is used ONLY to store the controller handle +// (`init`) or clone it out (`BusGuard::claim`) — a microsecond-scale window that +// never spans a poll. Ownership is single-caller by construction (one governor +// task; `init` runs before it), so `BusGuard::claim` takes ownership without +// blocking and bails on the (never-expected) contended case rather than spinning. +// `SpinNoPreempt` (not `SpinNoIrq`) keeps local IRQs enabled; the lock is never +// taken from an interrupt handler, so no re-entrant self-deadlock is possible. use ax_kspin::SpinNoPreempt as Mutex; use log::{info, warn}; use mmio_api::{MmioAddr, MmioRaw}; @@ -229,6 +236,7 @@ enum I2cErr { Timeout, } +#[derive(Clone)] struct Rk3xI2c { mmio: MmioRaw, } @@ -403,8 +411,71 @@ impl Rk3xI2c { // Public API — single global PMIC bus (mapped once) // --------------------------------------------------------------------------- +/// Storage for the initialised controller handle. The `SpinNoPreempt` mutex is +/// held only for the microsecond-scale window that stores it (`init`) or clones +/// it out ([`BusGuard::claim`]) — NEVER across a hardware poll. static CONTROLLER: Mutex> = Mutex::new(None); +/// Bus-ownership flag for the slow chip transactions. Unlike a `SpinNoPreempt` +/// guard, holding it does NOT disable preemption, so a `wait_ipd` poll (~100 ms +/// budget, longer if the bus is dead) can be preempted and never starves the +/// scheduler on that CPU — the blocking concern raised in review. +/// +/// Concurrency protocol: the PMIC bus has exactly one steady-state caller, the +/// single `cpufreq` governor task, and `init` runs once at boot before that task +/// exists — so contention does not occur by construction. The flag makes that +/// explicit and safe: [`BusGuard::claim`] tries to take ownership without +/// blocking and *bails* (returns `None` → the caller's safe no-op) if a +/// transaction is somehow already in flight, rather than spinning. A transaction +/// preempted mid-poll simply holds ownership across the gap; the hardware +/// transaction keeps running and resuming the status poll is idempotent. +static BUS_BUSY: AtomicBool = AtomicBool::new(false); + +/// RAII ownership of the PMIC bus for one (possibly multi-step) transaction. +/// Carries a clone of the controller handle so the slow poll runs with no +/// `SpinNoPreempt` guard held. Releases ownership on drop. +struct BusGuard { + i2c: Rk3xI2c, +} + +impl BusGuard { + /// Take exclusive bus ownership without blocking, then clone the controller + /// handle out under a microsecond-scale `CONTROLLER` lock. Returns `None` + /// if the bus is uninitialised or already owned (see the protocol note on + /// [`BUS_BUSY`]); in both cases the caller falls back to its safe no-op. + fn claim() -> Option { + if BUS_BUSY + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + warn!("pmic_i2c: bus already in use; skipping this transaction"); + return None; + } + // Brief preempt-disabled window: just clone the handle out, no poll. + let handle = CONTROLLER.lock().as_ref().cloned(); + match handle { + Some(i2c) => Some(Self { i2c }), + None => { + BUS_BUSY.store(false, Ordering::Release); + None + } + } + } +} + +impl Drop for BusGuard { + fn drop(&mut self) { + BUS_BUSY.store(false, Ordering::Release); + } +} + +impl core::ops::Deref for BusGuard { + type Target = Rk3xI2c; + fn deref(&self) -> &Rk3xI2c { + &self.i2c + } +} + /// Ungate the i2c0 PCLK + functional clock in the PMU CRU so the controller can /// generate SCL. Idempotent — the Rockchip CRU gate registers are write-masked /// (high 16 bits select which bits to write, low 16 are the values), and a `0` @@ -569,9 +640,9 @@ pub fn init() -> bool { /// or [`RK8603_BIG1_ADDR`]. `None` if the bus is uninitialised or the read /// fails. pub fn get_uv(chip: u8) -> Option { - let guard = CONTROLLER.lock(); - let i2c = guard.as_ref()?; - let vsel = i2c.read_reg(chip, VSEL_REG)?; + // Bus ownership (not a preempt-disabling lock) is held across the poll below. + let bus = BusGuard::claim()?; + let vsel = bus.read_reg(chip, VSEL_REG)?; Some(vsel_to_uv(vsel & VSEL_MASK)) } @@ -594,12 +665,11 @@ pub fn set_uv(chip: u8, target_uv: u32) -> bool { warn!("pmic_i2c: {target_uv} uV is not an exact VSEL step; refusing"); return false; }; - let guard = CONTROLLER.lock(); - let Some(i2c) = guard.as_ref() else { - warn!("pmic_i2c: not initialised; refusing set"); + let Some(bus) = BusGuard::claim() else { + warn!("pmic_i2c: not initialised or busy; refusing set"); return false; }; - i2c.set_vsel_verify(chip, vsel) + bus.set_vsel_verify(chip, vsel) } /// Safely lower a rail to `target_uv` by stepping **down** in ≤25 mV (4-LSB) @@ -625,13 +695,14 @@ pub fn set_uv_stepped(chip: u8, target_uv: u32) -> bool { return false; }; - let guard = CONTROLLER.lock(); - let Some(i2c) = guard.as_ref() else { - warn!("pmic_i2c: not initialised; refusing step"); + // One bus claim for the whole stepped transaction (read + the step writes), + // so it stays atomic w.r.t. other callers while never disabling preemption. + let Some(bus) = BusGuard::claim() else { + warn!("pmic_i2c: not initialised or busy; refusing step"); return false; }; - let Some(cur_vsel) = i2c.read_reg(chip, VSEL_REG).map(|v| v & VSEL_MASK) else { + let Some(cur_vsel) = bus.read_reg(chip, VSEL_REG).map(|v| v & VSEL_MASK) else { warn!("pmic_i2c: chip {chip:#x} current VSEL read failed; refusing step"); return false; }; @@ -651,7 +722,7 @@ pub fn set_uv_stepped(chip: u8, target_uv: u32) -> bool { let mut v = cur_vsel; while v > target_vsel { let next = v.saturating_sub(STEP_LSB).max(target_vsel); - if !i2c.set_vsel_verify(chip, next) { + if !bus.set_vsel_verify(chip, next) { warn!( "pmic_i2c: chip {chip:#x} step to VSEL {next:#x} ({} uV) failed; aborting at {} uV", vsel_to_uv(next), diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs b/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs index 60605b9ff1..1985b8cf03 100644 --- a/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs +++ b/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs @@ -81,17 +81,23 @@ //! pmic_spi::set_uv_stepped(675_000); // down-only to the OPP nominal //! ``` -use core::time::Duration; - -// PMIC access is slow (SPI register reads/writes that busy-wait on hardware), so -// the lock is held across the whole transaction. Use `SpinNoPreempt`, not -// `SpinNoIrq`: it keeps local IRQs ENABLED during the poll (so IRQ latency is not -// held hostage to a millisecond SPI transaction) while still disabling preemption -// so no task switch can interleave mid-transaction and corrupt the register -// sequence. This is sound because the lock is NEVER taken from an interrupt -// handler — every caller (`init`/`get_uv`/`set_uv*`/`force_write_dcdc2`) runs in -// the boot probe or the sleepable `cpufreq` governor task, so an IRQ arriving -// mid-transaction can never re-enter and self-deadlock. +use core::{ + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +// PMIC access is slow: an SPI register read/write busy-waits on the controller +// (~20 ms per transfer). Serialisation must NOT disable preemption across that +// poll, or the caller would stall task-switching on its CPU for the whole +// transaction (the scheduling-starvation risk raised in review). So the slow +// transactions are guarded by a non-preempt-disabling bus-ownership flag +// (`BUS_BUSY` / [`BusGuard`]); the `SpinNoPreempt` `Mutex` below is used ONLY to +// store the controller handle (`init`) or clone it out (`BusGuard::claim`) — a +// microsecond window that never spans a poll. Ownership is single-caller by +// construction (the A55 rail is programmed once at boot; the ring-only governor +// issues no dynamic A55 write), so `BusGuard::claim` bails on the never-expected +// contended case rather than spinning. `SpinNoPreempt` (not `SpinNoIrq`) keeps +// IRQs enabled; the lock is never taken from an interrupt handler. use ax_kspin::SpinNoPreempt as Mutex; use log::{info, warn}; use rdif_pinctrl::PinctrlDevice; @@ -262,17 +268,73 @@ const STEP_SETTLE_US: u64 = 150; // --------------------------------------------------------------------------- /// A mapped, initialized SPI2 controller bound to the RK806. +#[derive(Clone)] struct Rk806Spi { base: *mut u8, } -// The controller is reached only through `PMIC` (a `SpinNoPreempt` mutex), which -// serializes all access; the MMIO mapping is stable for the kernel lifetime. +// The controller handle is only ever stored/cloned under `PMIC` (a microsecond +// `SpinNoPreempt` window) and used while its owner holds `BUS_BUSY`; the MMIO +// mapping is stable for the kernel lifetime. unsafe impl Send for Rk806Spi {} -/// Global RK806/SPI2 handle, populated once by [`init`]. +/// Global RK806/SPI2 handle, populated once by [`init`]. The `SpinNoPreempt` +/// mutex is held only to store the handle or clone it out ([`BusGuard::claim`]), +/// never across an SPI poll. static PMIC: Mutex> = Mutex::new(None); +/// Non-preempt-disabling bus-ownership flag for the slow RK806 SPI transactions +/// (`rk806_read`/`rk806_write` poll the controller, ~20 ms per transfer). Holding +/// it does not disable preemption, so a transfer can be preempted and never +/// starves the scheduler on that CPU (the review concern). Single-caller by +/// construction (the A55 rail is programmed once at boot by `align_rail_voltages`; +/// the ring-only governor never issues a dynamic A55 voltage write), so +/// [`BusGuard::claim`] takes ownership without blocking and bails on the +/// never-expected contended case instead of spinning. +static BUS_BUSY: AtomicBool = AtomicBool::new(false); + +/// RAII ownership of the RK806 SPI bus for one transaction. Carries a clone of +/// the controller handle so the poll runs with no `SpinNoPreempt` guard held; +/// releases ownership on drop. +struct BusGuard { + dev: Rk806Spi, +} + +impl BusGuard { + /// Take bus ownership without blocking, then clone the handle out under a + /// microsecond `PMIC` lock. `None` if uninitialised or already owned. + fn claim() -> Option { + if BUS_BUSY + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + warn!("pmic_spi: bus already in use; skipping this transaction"); + return None; + } + let handle = PMIC.lock().as_ref().cloned(); + match handle { + Some(dev) => Some(Self { dev }), + None => { + BUS_BUSY.store(false, Ordering::Release); + None + } + } + } +} + +impl Drop for BusGuard { + fn drop(&mut self) { + BUS_BUSY.store(false, Ordering::Release); + } +} + +impl core::ops::Deref for BusGuard { + type Target = Rk806Spi; + fn deref(&self) -> &Rk806Spi { + &self.dev + } +} + impl Rk806Spi { #[inline] fn read(&self, off: usize) -> u32 { @@ -780,9 +842,8 @@ pub fn init() -> bool { /// Read the current `vdd_cpu_lit` (A55 rail) voltage in microvolts, or `None` /// if the controller is not initialized or the SPI read timed out. pub fn get_uv() -> Option { - let guard = PMIC.lock(); - let dev = guard.as_ref()?; - let sel = dev.rk806_read(RK806_BUCK2_ON_VSEL)?; + let bus = BusGuard::claim()?; + let sel = bus.rk806_read(RK806_BUCK2_ON_VSEL)?; let uv = vsel_to_uv(sel); info!("pmic_spi: A55 vdd_cpu_lit = {uv} uV (buck2 vsel {sel:#04x})"); Some(uv) @@ -800,11 +861,11 @@ pub fn get_uv() -> Option { // not call it. #[allow(dead_code)] pub fn set_uv(target_uv: u32) -> bool { - let guard = PMIC.lock(); - let Some(dev) = guard.as_ref() else { - warn!("pmic_spi: set_uv before init; ignored"); + let Some(bus) = BusGuard::claim() else { + warn!("pmic_spi: set_uv before init or busy; ignored"); return false; }; + let dev = &*bus; let Some(boot_sel) = dev.rk806_read(RK806_BUCK2_ON_VSEL) else { warn!("pmic_spi: set_uv could not read boot voltage; leaving rail untouched"); return false; @@ -823,11 +884,12 @@ pub fn set_uv(target_uv: u32) -> bool { /// is an accepted no-op (`true`). Any read-back mismatch aborts mid-descent and /// returns `false`, leaving the last verified selector in place. pub fn set_uv_stepped(target_uv: u32) -> bool { - let guard = PMIC.lock(); - let Some(dev) = guard.as_ref() else { - warn!("pmic_spi: set_uv_stepped before init; ignored"); + // One claim for the whole stepped transaction (read + step writes). + let Some(bus) = BusGuard::claim() else { + warn!("pmic_spi: set_uv_stepped before init or busy; ignored"); return false; }; + let dev = &*bus; let Some(cur_sel) = dev.rk806_read(RK806_BUCK2_ON_VSEL) else { warn!("pmic_spi: set_uv_stepped could not read current voltage; leaving rail untouched"); return false; @@ -879,11 +941,11 @@ pub fn force_write_dcdc2(target_uv: u32) -> bool { ); return false; } - let guard = PMIC.lock(); - let Some(dev) = guard.as_ref() else { - warn!("pmic_spi: force_write_dcdc2 before init; ignored"); + let Some(bus) = BusGuard::claim() else { + warn!("pmic_spi: force_write_dcdc2 before init or busy; ignored"); return false; }; + let dev = &*bus; let Some(sel) = uv_to_vsel(target_uv) else { warn!("pmic_spi: force_write_dcdc2 target {target_uv} uV not encodable"); return false;