diff --git a/components/axcpu/src/aarch64/init.rs b/components/axcpu/src/aarch64/init.rs index 4fc82a3f15..1318adf905 100644 --- a/components/axcpu/src/aarch64/init.rs +++ b/components/axcpu/src/aarch64/init.rs @@ -118,6 +118,19 @@ pub fn init_trap() { #[cfg(feature = "uspace")] { CNTKCTL_EL1.modify(CNTKCTL_EL1::EL0VCTEN::TrappedNone + CNTKCTL_EL1::EL0PCTEN::TrappedNone); + // Start this CPU's free-running PMU cycle counter and let EL0 read it, so + // `PMCCNTR_EL0` yields real cycle counts at both EL1 and EL0. This is the + // exact-frequency oracle the DVFS calibration reads in-kernel and that + // `cpuprobe`'s `mhz_pmc` reads from userspace (both were 0/trapping before, + // because these registers were only ever set on the lazy perf_event_open + // path). Guarded on PMUv3 being present; a live system-wide `perf stat -e + // cycles` may momentarily reset/disable this shared counter, which is fine + // for a boot/idle calibration. + if crate::pmu::probe().is_some() { + crate::pmu::init_cpu(); + crate::pmu::cycles::configure(false, false); + crate::pmu::cycles::enable(); + } barrier::isb(barrier::SY); } unsafe extern "C" { diff --git a/docs/rk3588-cpufreq.md b/docs/rk3588-cpufreq.md new file mode 100644 index 0000000000..92834225a1 --- /dev/null +++ b/docs/rk3588-cpufreq.md @@ -0,0 +1,62 @@ +# RK3588 CPU DVFS (cpufreq) + +Feature-gated ondemand CPU frequency/voltage scaling for the Orange Pi 5 Plus +(RK3588). It is off by default in the generic and QEMU builds — the +`ax-driver/rk3588-cpufreq` feature compiles the driver to no-ops there — and is +enabled on the Orange Pi board build. + +## What it does + +The RK3588 CPU clock is a voltage-coupled PVTPLL. An **ondemand** governor scales +each of the three CPU clusters (A55 little `cpu0-3`, two A76 big pairs `cpu4-5` / +`cpu6-7`) between OPPs by pairing an SCMI PVTPLL ring target with a PMIC rail +voltage: + +- **A76** uses the full voltage lever — its RK8602/RK8603 rails read back over I2C, + so every voltage write is confirmed and the SCMI clock rate is read back before an + OPP is committed. +- **A55** is **ring-only**: its RK806 rail cannot be read back (a MISO hardware + limit), so it stays on the boot-confirmed 675 mV rail and scales only its SCMI + ring. 675 mV over-volts every A55 ring ≤ 1008 MHz, so it can never undervolt. + +It is **fail-safe**: if either PMIC bus does not come up at boot, `GOV_READY` stays +`false` and every cluster is left on its boot OPP (no scaling). + +## Enable and build + +The feature is enabled in both Orange Pi 5 Plus build configs: + +- `test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml` + — the config the **board CI runner** actually builds, so the standard board + test entry compiles, boots and runs the governor on real hardware: + + ```bash + cargo xtask starry test board --board orangepi-5-plus + ``` + + Every case there boots this kernel; a governor-induced boot panic/hang trips the + cases' `panic` fail-regex, so the board CI regression-guards the feature. +- `os/StarryOS/configs/board/orangepi-5-plus.toml` — the general board build + template (for a full-board `max_cpu_num` build outside CI). + +To enable it in a custom build, add `"ax-driver/rk3588-cpufreq"` to that config's +`features` list. + +## Observable verification + +At boot (early init, before the console handoff) the driver logs the rail bring-up +and governor arming to the serial console: + +``` +cpufreq: A55 rail boot voltage = uV +cpufreq: A55 ->, A76 -> MHz +cpufreq: ondemand governor armed (both PMIC buses up) +``` + +Under load, per-cluster OPP transitions are logged +(`gov: A76b0 peak=% opp -> = MHz @ mV`), and the delivered +frequency can be read exactly with `cpuprobe`'s `mhz_pmc` (the PMU cycle counter is +enabled at boot): a busy cluster climbs toward its top OPP, an idle cluster sheds +back down one step at a time. The companion `apps/starry/sysbench-board` harness +(PR #1658) drives an all-core load and captures these numbers against a Linux +baseline. diff --git a/drivers/ax-driver/Cargo.toml b/drivers/ax-driver/Cargo.toml index 40c319d2f2..dedb501ecc 100644 --- a/drivers/ax-driver/Cargo.toml +++ b/drivers/ax-driver/Cargo.toml @@ -136,6 +136,12 @@ rockchip-soc = [ "dep:rdif-reset", "dep:rockchip-soc", ] +# RK3588 fixed-OPP-at-boot CPU DVFS (voltage-free rung): raises the CPU cluster +# clocks via the SCMI seam at probe time. Board-only policy; keep it opt-in. +rk3588-cpufreq = [ + "rockchip-soc", + "dep:arm-scmi-rs", +] rockchip-pm = ["rockchip-soc", "dep:rdif-power", "dep:rockchip-pm"] list-pci-devices = ["pci"] pci-list-devices = ["list-pci-devices"] diff --git a/drivers/ax-driver/src/lib.rs b/drivers/ax-driver/src/lib.rs index af85932093..a35b6d0e9e 100644 --- a/drivers/ax-driver/src/lib.rs +++ b/drivers/ax-driver/src/lib.rs @@ -95,6 +95,44 @@ pub mod usb; #[cfg(virtio_dev)] pub mod virtio; +/// RK3588 CPU DVFS ondemand governor, exposed as a stable, arch-neutral entry +/// the kernel can drive from a periodic task without knowing the SoC specifics. +/// +/// The governor's *policy + apply* live in the (arch-specific) cpufreq driver, +/// but its *loop* — sleeping between samples and reading the per-CPU busy +/// counters — cannot live in this crate: ax-driver sits below ax-task/ax-hal in +/// the dependency graph, so spawning a task here would be a cyclic dependency. +/// The kernel therefore owns the loop and calls [`cpufreq::governor_poll`] each +/// tick. When the DVFS feature is off these are no-ops so callers stay generic. +pub mod cpufreq { + #[cfg(feature = "rk3588-cpufreq")] + pub use crate::soc::rockchip::cpufreq::{ + calibrate_cluster, calibrate_wanted, governor_period_ms, governor_poll, governor_wanted, + }; + + /// Feature-off stub: no governor, so the kernel never spawns its task. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn governor_wanted() -> bool { + false + } + /// Feature-off stub. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn governor_period_ms() -> u64 { + 100 + } + /// Feature-off stub. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn governor_poll(_busy: &[u64]) {} + /// Feature-off stub: no calibration. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn calibrate_wanted() -> bool { + false + } + /// Feature-off stub. + #[cfg(not(feature = "rk3588-cpufreq"))] + pub fn calibrate_cluster(_cluster_idx: usize, _intended_cpu: usize) {} +} + #[cfg(feature = "pci")] pub use binding_info::PciIrqRequirement; pub use binding_info::{BindingInfo, BindingIrq, BindingIrqBinding, BindingIrqSource, FdtIrqSpec}; diff --git a/drivers/ax-driver/src/soc/mod.rs b/drivers/ax-driver/src/soc/mod.rs index fd40781641..7e45658c5b 100644 --- a/drivers/ax-driver/src/soc/mod.rs +++ b/drivers/ax-driver/src/soc/mod.rs @@ -15,8 +15,8 @@ #[cfg(feature = "pinctrl")] mod fixed_regulator; #[cfg(feature = "rockchip-soc")] -mod rockchip; -#[cfg(feature = "rockchip-dwmmc")] +pub(crate) mod rockchip; +#[cfg(any(feature = "rockchip-dwmmc", feature = "rk3588-cpufreq"))] pub mod scmi; #[cfg(feature = "starfive-soc")] mod starfive; diff --git a/drivers/ax-driver/src/soc/rockchip/cpufreq.rs b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs new file mode 100644 index 0000000000..16c7a19978 --- /dev/null +++ b/drivers/ax-driver/src/soc/rockchip/cpufreq.rs @@ -0,0 +1,1283 @@ +// Copyright 2025 The Axvisor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RK3588 CPU DVFS: SCMI clocks + PMIC rail-voltage alignment + ondemand governor. +//! +//! The RK3588 CPU clock is voltage-coupled — an SCMI clock id selects a PVTPLL +//! ring whose *delivered* frequency tracks the core rail voltage — and the SCMI +//! interface is frequency-only. Exact DVFS therefore needs BOTH levers together: +//! the SCMI ring *and* the matching rail voltage. This driver does three things, +//! in order: +//! +//! 1. **Set each cluster clock** to its boot target over the board-proven SCMI +//! seam ([`set_and_verify`]). The three CPU domains and their SCMI clock ids +//! (ground truth `orangepi5plus.dts`: `cpu@0..300` → ``, `cpu@400/500` +//! → ``, `cpu@600/700` → ``): +//! +//! | cluster | SCMI clock id | boot target (MHz) | +//! |----------------|---------------|-------------------| +//! | A55 (little) | 0 | 1008 | +//! | A76 big pair 0 | 2 | 1200 | +//! | A76 big pair 1 | 3 | 1200 | +//! +//! 2. **Align each rail voltage to the OPP** ([`align_rail_voltages_to_opp`]). +//! Boot firmware leaves the rails high (~800 mV), so the coupled clock +//! overshoots the SCMI target until each rail is lowered to its OPP-nominal +//! (0.675 V for these OPPs on the standard SKU). Lowering cannot undervolt: the +//! coupled clock tracks the rail down in lockstep. NOTE the industrial +//! RK3588J/M SKU (selected by the `specification_serial_number` nvmem cell) +//! puts these OPPs at 0.75 V; the PMIC modules floor at 0.675 V, so gate the +//! nominal on the SKU cell before running this on a J/M part. +//! +//! 3. **Hand off to the ondemand governor** ([`governor_poll`]). Once both +//! CPU-rail PMIC buses are up, a dynamic governor scales each cluster's OPP to +//! match load (see the governor section at the end of this file). +//! +//! Registration and ordering: a `PostKernel` / `DEFAULT` rdrive probe, so the CRU + +//! SCMI providers (registered at `CLK` priority) are already live, and it runs +//! inside `devices::probe_all_devices()` — **before** `start_secondary_cpus()` — +//! so the A76 clusters are reclocked while no core is scheduled on them (the live +//! A55 id-0 switch is BL31's glitch-free path). It binds to the CPU nodes rather +//! than `arm,scmi-smc` (which the SCMI driver already owns; a second driver on +//! that node would never get an `on_probe`), and applies exactly once via a +//! one-shot guard because several `cpu@*` nodes match. + +use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; + +use fdt_edit::Phandle; +use log::{info, warn}; + +use crate::{probe::OnProbeError, register::ProbeFdt, soc::scmi}; + +/// SCMI clock id of the A55 (little) cluster — cpu0..3. +const A55_CLK_ID: u32 = 0; +/// SCMI clock ids of the two A76 (big) cluster pairs — cpu4/5 and cpu6/7. Both +/// must be set; they cover different core pairs. +const A76_CLK_IDS: [u32; 2] = [2, 3]; + +/// A55 target and hard ceiling: the top OPP still on the 816 MHz boot voltage +/// row. `set_clock_rate` must never be driven above this for the A55 cluster. +const A55_MAX_HZ: u64 = 1_008_000_000; +/// A76 target and hard ceiling: the top OPP still on the 816 MHz boot voltage +/// row. `set_clock_rate` must never be driven above this for an A76 cluster. +const A76_MAX_HZ: u64 = 1_200_000_000; + +/// One-shot guard: several `cpu@*` nodes match, but the reclock runs once. +static APPLIED: AtomicBool = AtomicBool::new(false); + +crate::model_register!( + name: "RK3588 CPU DVFS SCMI", + level: ProbeLevel::PostKernel, + priority: ProbePriority::DEFAULT, + probe_kinds: &[ + ProbeKind::Fdt { + compatibles: &["arm,cortex-a55", "arm,cortex-a76"], + on_probe: probe + } + ], +); + +fn probe(_probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { + // Several CPU nodes match this driver; only the first invocation reclocks. + if APPLIED.swap(true, Ordering::AcqRel) { + return Ok(()); + } + + // The `scmi::*` helpers ignore the phandle (single global agent); pass a + // dummy so we do not depend on parsing the node's clock specifier. + let phandle = Phandle::from(0u32); + + // Safety preflight (read-only): confirm this firmware actually services the + // CPU-cluster clocks before touching any of them. `describe_rates` changes + // no state, so this cannot hang or perturb the clocks; treat its result as + // accept/reject only. If any target id is rejected, leave every cluster at + // its boot rate and bail — there is no raw-CRU fallback here. + for id in [A55_CLK_ID, A76_CLK_IDS[0], A76_CLK_IDS[1]] { + if scmi::describe_rates(phandle, id).is_none() { + warn!( + "cpufreq: SCMI does not service CPU cluster clock id {id}; leaving all CPU \ + clusters at their boot rate (no DVFS applied)" + ); + return Ok(()); + } + } + + let a55_before = read_mhz(phandle, A55_CLK_ID); + if !set_and_verify(phandle, A55_CLK_ID, A55_MAX_HZ, A55_MAX_HZ) { + warn!("cpufreq: A55 reclock did not verify; stopping (A76 left at boot rate)"); + return Ok(()); + } + let a55_after = read_mhz(phandle, A55_CLK_ID); + + let a76_before = read_mhz(phandle, A76_CLK_IDS[0]); + for id in A76_CLK_IDS { + if !set_and_verify(phandle, id, A76_MAX_HZ, A76_MAX_HZ) { + warn!("cpufreq: A76 cluster clock id {id} reclock did not verify; stopping"); + return Ok(()); + } + } + let a76_after = read_mhz(phandle, A76_CLK_IDS[0]); + + info!("cpufreq: A55 {a55_before}->{a55_after}, A76 {a76_before}->{a76_after} MHz"); + + // The CPU clock is voltage-coupled (proven on-board: at a fixed SCMI clock the + // A76 runs ~1.19 GHz @675 mV but ~1.49 GHz @800 mV), so the clocks set above + // overshoot while each rail sits at its ~800 mV boot value. Lower each rail to + // its OPP nominal to pull the coupled clock onto the exact requested rate. Runs + // LAST — after the SCMI clock is confirmed at target — so the down-shift is the + // final step (matches Linux's reduce-freq-then-voltage order for + // down-transitions), and only on the full-success path above. + align_rail_voltages_to_opp(); + + // Hand off to the dynamic ondemand governor. It cannot + // live entirely in this crate — ax-driver sits *below* ax-task/ax-hal in the + // dependency graph (they pull ax-driver back in via axplat-dyn), so a + // task-spawning loop here would be a cyclic dep. Instead this driver exposes + // the pure policy+apply (`governor_poll`, see the governor section at the end + // of this file) and the kernel drives it from a periodic sleepable task. The + // voltage lever above armed `GOV_READY` iff both PMIC buses came up. + + Ok(()) +} + +/// Programs `clock_id` to `target`, but never above `ceiling` (the hard cap on +/// the boot voltage row), then verifies the platform actually applied it. +/// Returns `true` only when the read-back matches the request. +/// +/// `target == ceiling` for the boot targets; the clamp is defense in depth so a future +/// edit can never push a cluster past its boot-voltage-safe ceiling. A rejected +/// set or a deviating read-back is reported and returns `false` so the caller +/// stops rather than leaving a partial/phantom state — the board stays on +/// whatever rate the firmware last confirmed. +fn set_and_verify(phandle: Phandle, clock_id: u32, target: u64, ceiling: u64) -> bool { + if target > ceiling { + warn!( + "cpufreq: refusing to set clock id {clock_id} to {target} Hz (above boot-safe ceiling \ + {ceiling} Hz)" + ); + return false; + } + if scmi::set_clock_rate(phandle, clock_id, target).is_none() { + warn!("cpufreq: SCMI rejected clock id {clock_id} set to {target} Hz; left unchanged"); + return false; + } + match scmi::clock_rate(phandle, clock_id) { + Some(applied) if applied == target => true, + Some(applied) if applied > ceiling => { + warn!( + "cpufreq: clock id {clock_id} read back {applied} Hz ABOVE boot-safe ceiling \ + {ceiling} Hz (requested {target} Hz); stopping" + ); + false + } + Some(applied) => { + warn!( + "cpufreq: clock id {clock_id} read back {applied} Hz, requested {target} Hz; \ + stopping" + ); + false + } + None => { + warn!("cpufreq: could not read back clock id {clock_id} after set; stopping"); + false + } + } +} + +/// Current rate of `clock_id` in MHz, or 0 if it cannot be read (logging only). +fn read_mhz(phandle: Phandle, clock_id: u32) -> u64 { + scmi::clock_rate(phandle, clock_id).unwrap_or(0) / 1_000_000 +} + +// =========================================================================== +// CPU-rail voltage alignment (the voltage half of the coupled clock) +// =========================================================================== + +/// Master gate for the PMIC **writes**. While `false`, [`align_rail_voltages_to_opp`] +/// only *reads and logs* each rail's boot voltage (zero PMIC writes). Flipped to +/// `true` after the read-only board pass confirmed the A76 rails read the true +/// 800 mV boot voltage (i2c0 bring-up: ungate + reset + pinmux). Even with this +/// on, a rail is only lowered when its own read is trustworthy (see the A55 gate +/// below — its spi2/RK806 read is not up yet, so it is skipped). +const APPLY_RAIL_VOLTAGE: bool = true; + +/// OPP-nominal core voltage for the boot targets on the **standard** SKU: the +/// 675 mV row shared by the 816/1008/1200 MHz OPPs (board-confirmed from the +/// `cluster*-opp-table` `opp-microvolt`). The A55 1008 OPP and both A76 1200 OPPs +/// all sit here. +/// +/// NOTE: the industrial RK3588J/M SKU puts these OPPs at 750 mV (`opp-j-m-*`). The +/// PMIC modules floor at 675 mV, so a 675 mV target would *under*-volt a J/M part. +/// This board is the standard SKU; if this driver is ever run on a J/M board, gate +/// this constant on the `specification_serial_number` nvmem SKU cell first. +const A76_NOMINAL_UV: u32 = 675_000; +const A55_NOMINAL_UV: u32 = 675_000; + +/// One-shot A55 MOSI/write diagnostic — **OFF by default**. The RK806 read path is +/// dead (returns a bogus 0x00), so an A55 write cannot be validated by read-back. +/// When enabled this force-writes DCDC2 to the bounded-safe A55 OPP nominal and +/// relies on cpuprobe observing whether the A55 frequency drops — the only way to +/// learn if MOSI/writes physically reach the RK806 when reads do not. But that is +/// an *unconfirmable* A55 rail write, which is exactly the boundary the ring-only +/// A55 policy exists to avoid (the governor never writes the A55 rail; A55 keeps +/// its boot voltage, which over-volts every ring <= 1008 MHz). Leave it `false` in +/// production; flip to `true` only for a deliberate board-side MOSI reachability +/// probe. The write is clamped to the A55 boot-safe row inside the PMIC module. +const A55_FORCE_WRITE_TEST: bool = false; + +/// Read (and, once validated, lower) the three CPU-cluster rails to their OPP +/// nominal so the voltage-coupled clock lands on the exact requested frequency. +/// +/// Order matters: the A76 clusters are reclocked before `start_secondary_cpus()` +/// so no core is scheduled on them — their voltage is lowered first. The A55 rail +/// feeds the live boot core, so it is lowered last, and only via the stepped path +/// (the voltage-coupled clock tracks the rail down with no undervolt transient). +fn align_rail_voltages_to_opp() { + use super::{pmic_i2c, pmic_spi}; + + // --- A76 big0/big1 rails: RK8602 @0x42 / RK8603 @0x43 over I2C bus0 --- + let a76_ok = pmic_i2c::init(); + if a76_ok { + for (name, chip) in [ + ("big0", pmic_i2c::RK8602_BIG0_ADDR), + ("big1", pmic_i2c::RK8603_BIG1_ADDR), + ] { + match pmic_i2c::get_uv(chip) { + Some(uv) => info!("cpufreq: A76 {name} rail boot voltage = {uv} uV"), + None => warn!("cpufreq: A76 {name} rail voltage read failed"), + } + } + } else { + warn!("cpufreq: A76 PMIC (I2C) init failed; A76 left at boot voltage"); + } + + // --- A55 (little) rail: RK806 DCDC2 over SPI2 --- + let a55_ok = pmic_spi::init(); + if a55_ok { + match pmic_spi::get_uv() { + Some(uv) => info!("cpufreq: A55 rail boot voltage = {uv} uV"), + None => warn!("cpufreq: A55 rail voltage read failed"), + } + } else { + warn!("cpufreq: A55 PMIC (SPI) init failed; A55 left at boot voltage"); + } + + if !APPLY_RAIL_VOLTAGE { + info!("cpufreq: rail-voltage alignment is READ-ONLY this build (no PMIC writes)"); + return; + } + + // --- Apply: stepped, down-only, read-back-verified lower to OPP nominal. --- + if a76_ok { + let b0 = pmic_i2c::set_uv_stepped(pmic_i2c::RK8602_BIG0_ADDR, A76_NOMINAL_UV); + let b1 = pmic_i2c::set_uv_stepped(pmic_i2c::RK8603_BIG1_ADDR, A76_NOMINAL_UV); + info!("cpufreq: A76 rails -> {A76_NOMINAL_UV} uV (big0 ok={b0}, big1 ok={b1})"); + } + // A55 (spi2/RK806): only lower it if the current-voltage read is trustworthy. + // `set_uv_stepped` reads the rail before stepping down, so we must never lower a + // rail we cannot read. The spi2/RK806 read path is not up yet (it returns a bogus + // 0x00 == 500 mV), so skip the A55 write until that bring-up completes rather than + // act on a false reading. A real A55 boot voltage is in [675 mV, 950 mV] per the + // RK806 DCDC2 range and the cluster0 OPP table. + match if a55_ok { pmic_spi::get_uv() } else { None } { + Some(v) if (675_000..=950_000).contains(&v) => { + let a55 = pmic_spi::set_uv_stepped(A55_NOMINAL_UV); + info!("cpufreq: A55 rail {v} -> {A55_NOMINAL_UV} uV (ok={a55})"); + } + other => warn!( + "cpufreq: A55 boot voltage not trustworthy ({other:?} uV); skipping A55 write until \ + spi2/RK806 bring-up completes (A76 unaffected)" + ), + } + + // A55 MOSI/write diagnostic: force-write DCDC2 to the bounded-safe nominal and + // let cpuprobe reveal whether writes reach the RK806 even though reads don't. + if A55_FORCE_WRITE_TEST && a55_ok { + let ok = pmic_spi::force_write_dcdc2(A55_NOMINAL_UV); + info!( + "cpufreq: A55 MOSI/write test -> {A55_NOMINAL_UV} uV (write xfer_ok={ok}); watch A55 \ + cpuprobe (req=0..3) for a freq drop if the write reached" + ); + } + + // Arm the dynamic governor as long as the A76 I2C rail came up. The governor's + // only dynamic PMIC writes are the A76 rails (RK8602/RK8603, read-back verified + // over I2C); the A55 is ring-only — it scales solely via its SCMI PVTPLL ring + // and the governor never writes the A55 rail (see `Cluster::voltage_managed`). + // So a flaky SPI2/RK806 bring-up (a55_ok=false) must NOT disable the fully + // working A76 DVFS. When a55_ok is false the one-shot boot alignment above was + // skipped, so A55 keeps its boot rail voltage; that only ever OVER-volts the + // ring-only A55 rungs (<= 1008 MHz), never undervolts, so ring scaling stays + // safe. If the A76 I2C bus itself failed, leave every cluster on the boot OPP + // the SCMI reclock already set (the safe fixed-boot state). + if GOVERNOR_ENABLE && a76_ok { + GOV_READY.store(true, Ordering::Release); + info!("cpufreq: ondemand governor armed (A76 I2C up; a55_spi={a55_ok}, A55 ring-only)"); + } else if GOVERNOR_ENABLE { + warn!( + "cpufreq: ondemand governor NOT armed (a76_pmic={a76_ok}); clusters stay on boot OPP" + ); + } +} + +// =========================================================================== +// Dynamic ondemand governor +// =========================================================================== +// +// The voltage lever above pins each cluster at a single boot OPP. This governor +// makes DVFS *dynamic*: it samples per-CPU busy time and moves each cluster's +// OPP up and down to track load, the way Linux's `ondemand`/`schedutil` do. +// +// Split by cost, exactly like Linux: the *accounting* is a cheap per-CPU counter +// bumped in the scheduler tick (`ax_task::cpu_busy_ticks`, fed by the non-idle +// branch of `scheduler_timer_tick`), but the *apply* is slow and SLEEPS — an +// SCMI clock set is an SMC into BL31, and the paired PMIC write is an I2C/SPI +// transaction with a millisecond voltage ramp. Neither may run in the tick +// handler, so the decision+apply live in a periodic sleepable kernel task. This +// is exactly why Linux's old ondemand used a deferred timer and schedutil kicks +// a kthread rather than reprogramming the OPP inline in the tick. + +/// Master gate for the dynamic governor. When `false`, each cluster simply stays +/// on the fixed boot OPP the voltage alignment left it on. +const GOVERNOR_ENABLE: bool = true; + +/// An operating performance point: the SCMI ring target to program, the rail +/// voltage to pair with it, and the frequency that combination actually delivers +/// (`mhz`, board-measured — see the calibration section). Because the PVTPLL is +/// voltage-coupled, the delivered `mhz` is generally NOT the `ring_khz`; the +/// governor reports `mhz`. +#[derive(Clone, Copy)] +struct Opp { + ring_khz: u32, + uv: u32, + mhz: u32, +} + +/// A76 (big) OPP ladder, low→high, from the on-board calibration sweep. The clock +/// is voltage-coupled, so this ladder is a HYBRID: below the 675 mV exact point it +/// scales the SCMI ring (408/816/1200 @ 675 mV land on target); above it, it holds +/// the ring at 1200 and raises the *voltage* — the delivered freq climbs while +/// staying over-volted (each rung's voltage exceeds the delivered freq's DT +/// nominal, so never an undervolt). Scaling the ring instead (e.g. ring 1608 @ +/// 762.5 mV) over-delivers ~1733 MHz = ~125 mV of undervolt (measured), so it is +/// avoided. Top rung 1725 MHz @ 925 mV is the calibration sweep's safe maximum +/// (over-volted ~110 mV vs the delivered freq's DT nominal); board-validated +/// all-core (threads=8) with no PSU brownout. +const A76_OPPS: &[Opp] = &[ + Opp { + ring_khz: 408_000, + uv: 675_000, + mhz: 408, + }, + Opp { + ring_khz: 816_000, + uv: 675_000, + mhz: 816, + }, + Opp { + ring_khz: 1_200_000, + uv: 675_000, + mhz: 1189, + }, + Opp { + ring_khz: 1_200_000, + uv: 725_000, + mhz: 1318, + }, + Opp { + ring_khz: 1_200_000, + uv: 800_000, + mhz: 1491, + }, + Opp { + ring_khz: 1_200_000, + uv: 850_000, + mhz: 1592, + }, + Opp { + ring_khz: 1_200_000, + uv: 925_000, + mhz: 1725, + }, +]; + +/// A55 (little) OPP ladder, low→high. Unlike the A76 ladder this is **ring-only**: +/// every rung sits on the boot-confirmed 675 mV rail and only the SCMI PVTPLL ring +/// moves. The A55 rail is RK806 DCDC2 over SPI2, whose *read* path is a hardware +/// scope-wall — writes reach the chip (proven by the rail-alignment freq drop) but +/// MISO never feeds the shift register, so a write cannot be read back to confirm +/// the rail actually reached target. Scaling the A55 voltage on such an unconfirmable +/// write could commit an OPP whose rail never arrived; rather than risk that, the +/// little cluster forgoes the voltage lever and stays on 675 mV, which board +/// measurement proves delivers the full 1008 MHz ring exactly. 675 mV over-volts +/// every ring at or below 1008, so no A55 rung can undervolt regardless of the PMIC +/// (see [`Cluster::voltage_managed`]). The big clusters keep the voltage lever +/// because their RK8602/RK8603 rails read back over I2C. Restoring the higher +/// voltage-scaled A55 rungs (1212–1523 MHz) is gated on a trusted RK806 write-back +/// (an oscilloscope-level MISO fix, or a delivered-frequency oracle confirmation). +const A55_OPPS: &[Opp] = &[ + Opp { + ring_khz: 408_000, + uv: 675_000, + mhz: 408, + }, + Opp { + ring_khz: 816_000, + uv: 675_000, + mhz: 816, + }, + Opp { + ring_khz: 1_008_000, + uv: 675_000, + mhz: 1021, + }, +]; + +/// Index into each ladder of the boot OPP the voltage lever leaves the cluster on: +/// A55 1008 MHz and A76 1200 MHz are both element 2 (the 675 mV rung). The governor +/// starts tracking here so its first move is relative to the known boot state; from +/// idle it decays down the ring-scaled rungs and under load climbs the voltage ones. +const BOOT_OPP_IDX: usize = 2; + +/// The three DVFS domains (one little cluster, two big pairs). +#[derive(Clone, Copy)] +enum Cluster { + A55, + Big0, + Big1, +} + +impl Cluster { + fn opps(self) -> &'static [Opp] { + match self { + Cluster::A55 => A55_OPPS, + _ => A76_OPPS, + } + } + + /// SCMI clock id feeding this domain's PVTPLL ring. + fn clock_id(self) -> u32 { + match self { + Cluster::A55 => A55_CLK_ID, + Cluster::Big0 => A76_CLK_IDS[0], + Cluster::Big1 => A76_CLK_IDS[1], + } + } + + /// CPUs whose busy time this domain aggregates (RK3588 topology: cpu0-3 A55, + /// cpu4-5 big pair 0, cpu6-7 big pair 1). + fn cpus(self) -> core::ops::Range { + match self { + Cluster::A55 => 0..4, + Cluster::Big0 => 4..6, + Cluster::Big1 => 6..8, + } + } + + fn name(self) -> &'static str { + match self { + Cluster::A55 => "A55", + Cluster::Big0 => "A76b0", + Cluster::Big1 => "A76b1", + } + } + + /// Set this domain's rail to `uv`. A76 uses the read-back-verified I2C + /// regulator; A55 uses the bounded force-write (its RK806 read is a + /// scope-wall, but writes reach the chip — proven by the rail-alignment freq drop). + /// Both PMIC helpers clamp to their rail's safe envelope internally. + /// + /// Only ever called for a [`voltage_managed`](Self::voltage_managed) domain — + /// the A55 arm remains for the one-time boot alignment path but the governor + /// never drives it, so no dynamic (unconfirmable) A55 SPI write is issued. + fn set_voltage(self, uv: u32) -> bool { + use super::{pmic_i2c, pmic_spi}; + match self { + Cluster::A55 => pmic_spi::force_write_dcdc2(uv), + Cluster::Big0 => pmic_i2c::set_uv(pmic_i2c::RK8602_BIG0_ADDR, uv), + Cluster::Big1 => pmic_i2c::set_uv(pmic_i2c::RK8603_BIG1_ADDR, uv), + } + } + + /// Whether the governor may scale this domain's rail voltage dynamically. + /// + /// The A76 rails (RK8602/RK8603 over I2C) read back, so each write is confirmed + /// before the clock is allowed to follow it up — they get the full voltage lever. + /// The A55 rail (RK806 DCDC2 over SPI2) cannot be read back (a MISO hardware + /// scope-wall), so its voltage is programmed exactly once at init — the down-only, + /// boot-safe 675 mV alignment — and never scaled dynamically. The A55 ladder is + /// therefore ring-only and every A55 rung stays over-volted (675 mV supports the + /// whole ≤1008 MHz ring), which is undervolt-safe no matter what the unconfirmable + /// SPI write actually did. This per-cluster split keeps the readback-verified A76 + /// DVFS while removing every dynamic A55 voltage write. + fn voltage_managed(self) -> bool { + !matches!(self, Cluster::A55) + } +} + +/// One of the two hardware levers an OPP transition programs. Naming them makes +/// the safety-critical apply *ordering* assertable by a host test with no +/// hardware (see [`apply_step_order`]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum ApplyStep { + /// Program the rail voltage (PMIC I2C/SPI write). + Voltage, + /// Program the SCMI PVTPLL ring (clock rate). + Clock, +} + +/// Ordering policy for [`apply_opp`], pure so a host test can pin it: to keep the +/// voltage-coupled clock from ever overshooting its rail, an UPSHIFT raises the +/// voltage before the clock (clock follows up), and a DOWNSHIFT lowers the clock +/// before the voltage (clock follows down). +const fn apply_step_order(going_up: bool) -> [ApplyStep; 2] { + if going_up { + [ApplyStep::Voltage, ApplyStep::Clock] + } else { + [ApplyStep::Clock, ApplyStep::Voltage] + } +} + +/// Transactional core of [`apply_opp`], pure so a host test can drive every +/// success/failure combination. Runs the ordered steps, STOPS at the first failed +/// step (so the second lever is never moved after the first failed), and returns +/// `true` only when BOTH steps are confirmed. `run(step)` performs one hardware +/// step and reports whether it was confirmed. +fn run_apply_steps(order: [ApplyStep; 2], mut run: impl FnMut(ApplyStep) -> bool) -> bool { + for step in order { + if !run(step) { + return false; + } + } + true +} + +/// Apply an OPP to a domain as a matched (voltage, frequency) pair, ordered so +/// the voltage-coupled clock never overshoots its rail: +/// - going UP: raise voltage first, then the SCMI ring (clock follows up); +/// - going DOWN: lower the SCMI ring first, then voltage (clock follows down). +/// +/// TRANSACTIONAL: stops at the first failed step instead of falling through to +/// the second (which could otherwise create exactly the freq-high/voltage-low +/// window this ordering exists to prevent), and returns `true` only when BOTH +/// steps are confirmed. The caller ([`governor_poll`]) must commit its software +/// OPP index only on `true`; on `false` it must keep tracking the last +/// confirmed index so the next poll retries from there. The ordering + +/// stop-on-failure logic is the pure, host-tested [`apply_step_order`] + +/// [`run_apply_steps`]; only the hardware writes and their logs live here. +/// +/// Each step is CONFIRMED, not just requested: the clock step reads the SCMI rate +/// back and requires it to equal the request (a SET ack alone is not proof the ring +/// switched), and the A76 voltage step reads back over I2C. So "ring set succeeds" +/// below means the ring is read-back-confirmed at the new rate — a downshift only +/// reaches its voltage-lower step once the clock is verified already lowered. +/// +/// No-undervolt argument (both failure points, each direction): +/// - UPSHIFT, voltage step fails: the ring set is skipped entirely, so +/// nothing changed — old freq + old voltage, still a valid, previously +/// confirmed pairing. +/// - UPSHIFT, voltage step succeeds but the ring set fails: the rail is +/// already at (or above) `opp.uv` while the ring is still at its old, +/// lower value — over-volted for whatever it is currently delivering, +/// never under-volted. +/// - DOWNSHIFT, ring step fails: the voltage lower is skipped entirely, so +/// nothing changed — old freq + old voltage, still valid. +/// - DOWNSHIFT, ring step succeeds but the voltage step fails: the ring is +/// already at its new, lower value while the rail is still at its old, +/// higher voltage — over-volted for the new (lower) clock, never +/// under-volted. +#[must_use] +fn apply_opp(cluster: Cluster, opp: Opp, going_up: bool) -> bool { + let phandle = Phandle::from(0u32); + let hz = opp.ring_khz as u64 * 1_000; + run_apply_steps(apply_step_order(going_up), |step| match step { + ApplyStep::Voltage => { + // Ring-only clusters (A55) never scale voltage dynamically: their rail + // was pinned once at init to the boot-safe 675 mV row (down-only, and + // over-volted for every ring ≤ 1008) and cannot be read back to confirm + // a dynamic write. Treat the voltage step as a confirmed no-op so the + // ordered/transactional apply logic is unchanged while no unconfirmable + // SPI write is ever issued and the clock (already within the fixed rail's + // envelope) can safely follow. See [`Cluster::voltage_managed`]. + if !cluster.voltage_managed() { + return true; + } + if cluster.set_voltage(opp.uv) { + return true; + } + if going_up { + warn!( + "cpufreq: {} upshift to {} mV failed; leaving clock id {} unchanged (no \ + undervolt: old freq stays paired with old voltage)", + cluster.name(), + opp.uv / 1_000, + cluster.clock_id() + ); + } else { + warn!( + "cpufreq: {} clock already lowered to {} Hz but voltage write to {} mV \ + failed; not committing this OPP (safe: over-volted for the new, lower clock, \ + never under-volted)", + cluster.name(), + hz, + opp.uv / 1_000 + ); + } + false + } + ApplyStep::Clock => { + let cid = cluster.clock_id(); + // A SCMI CLOCK_RATE_SET ack is NOT proof the PVTPLL ring actually + // switched — read the rate back and require it to equal the request + // before treating the clock step as confirmed (same read-back the boot + // `set_and_verify` does). This is what makes a DOWNSHIFT safe: the + // voltage step that follows only runs once the clock is CONFIRMED at its + // new, lower rate, so we can never lower the rail under a still-high + // clock (the high-freq/low-voltage window the ordering exists to prevent). + let set_ok = scmi::set_clock_rate(phandle, cid, hz).is_some(); + let applied = if set_ok { + scmi::clock_rate(phandle, cid) + } else { + None + }; + if set_ok && applied == Some(hz) { + return true; + } + if going_up { + warn!( + "cpufreq: {} upshift: SCMI clock id {cid} not confirmed at {hz} Hz \ + (set_ok={set_ok}, read_back={applied:?}); not committing this OPP (safe: \ + voltage already raised, over-volted for the old, lower clock, never \ + under-volted)", + cluster.name() + ); + } else { + warn!( + "cpufreq: {} downshift: SCMI clock id {cid} not confirmed at {hz} Hz \ + (set_ok={set_ok}, read_back={applied:?}); leaving voltage unchanged (no \ + undervolt: clock not confirmed lowered, old freq stays paired with old \ + voltage)", + cluster.name() + ); + } + false + } + }) +} + +/// Period, in ms, the kernel governor task sleeps between [`governor_poll`]s. +const GOV_PERIOD_MS: u64 = 100; +/// Busy% at or above which a domain jumps straight to its top OPP (ondemand's +/// signature fast attack: respond to a load spike in one step). +const UP_THRESHOLD_PCT: u64 = 80; +/// Busy% below which a domain steps down one OPP (slow decay: shed frequency +/// gradually so a brief idle dip does not collapse a still-busy workload). +const DOWN_THRESHOLD_PCT: u64 = 30; + +/// Set once the voltage lever confirmed both PMIC buses are up. Until then the +/// governor must not move any rail (see `align_rail_voltages_to_opp`). +static GOV_READY: AtomicBool = AtomicBool::new(false); + +/// Per-CPU busy-tick value from the previous poll (RK3588 has 8 cores). Only the +/// single governor task ever touches these, so `Relaxed` is sufficient. +static LAST_BUSY: [AtomicU64; 8] = [const { AtomicU64::new(0) }; 8]; + +/// Per-cluster current OPP index (A55, big0, big1), starting on the boot OPP the +/// voltage lever pinned. +static IDX: [AtomicUsize; 3] = [const { AtomicUsize::new(BOOT_OPP_IDX) }; 3]; + +/// Cleared until the first [`governor_poll`] has recorded a busy baseline. The +/// first call must only prime `LAST_BUSY`, not decide: its delta is measured +/// from zero and would otherwise fold in every busy tick accumulated since boot, +/// spuriously pegging every cluster to its top OPP for one window. +static PRIMED: AtomicBool = AtomicBool::new(false); + +/// Monotonic timestamp (ns) of the previous [`governor_poll`], so each poll can +/// divide the busy-tick delta by the ACTUAL elapsed window rather than a fixed +/// nominal one. The governor task can be woken late (slow SCMI/PMIC transitions, +/// scheduling latency under load), which stretches the real window well past +/// `GOV_PERIOD_MS`; dividing by the nominal window then over-reports load and +/// would spuriously peg a merely-moderate cluster to its top OPP. +static LAST_POLL_NANOS: AtomicU64 = AtomicU64::new(0); + +/// Nanoseconds per scheduler tick (`cpu_busy_ticks` unit): TICKS_PER_SEC = 100 in +/// the kernel, so one tick is 10 ms. +const NANOS_PER_TICK: u64 = 10_000_000; + +/// Whether the dynamic governor should run: enabled at compile time *and* armed +/// by the voltage lever (both PMIC buses up). The kernel checks this once before +/// spawning its periodic governor task, so a failed PMIC bring-up leaves every +/// cluster safely on the boot OPP instead of being scaled with no voltage lever. +pub fn governor_wanted() -> bool { + GOVERNOR_ENABLE && GOV_READY.load(Ordering::Acquire) +} + +/// Period, in milliseconds, the kernel governor task should sleep between calls +/// to [`governor_poll`]. +pub fn governor_period_ms() -> u64 { + GOV_PERIOD_MS +} + +/// Pure ondemand policy, split out of [`governor_poll`] so the up/down/hold/prime +/// decision is host-testable with no hardware or tick counters. Given each core's +/// busy% over the last window (each already clamped to `0..=100`), the cluster's +/// current OPP index `cur`, the ladder length, and whether this is the priming +/// poll, return the next OPP index: +/// * priming poll, an empty cluster, or an empty ladder -> hold (no change); +/// * any core at/above [`UP_THRESHOLD_PCT`] -> jump straight to the top OPP +/// (ondemand fast-attack; a cluster shares one clock, so its busiest core +/// drives it — averaging would bury a single saturated thread); +/// * every core below [`DOWN_THRESHOLD_PCT`] -> shed exactly one step (never +/// below index 0); +/// * otherwise hold. +fn next_opp_idx(core_pcts: &[u64], cur: usize, ladder_len: usize, priming: bool) -> usize { + if priming || core_pcts.is_empty() || ladder_len == 0 { + return cur; + } + let any_core_high = core_pcts.iter().any(|&p| p >= UP_THRESHOLD_PCT); + let all_cores_low = core_pcts.iter().all(|&p| p < DOWN_THRESHOLD_PCT); + if any_core_high { + ladder_len - 1 + } else if all_cores_low && cur > 0 { + cur - 1 + } else { + cur + } +} + +/// One ondemand iteration, called periodically by the kernel governor task with +/// a fresh snapshot of every CPU's cumulative busy-tick counter +/// (`ax_task::cpu_busy_ticks`). Scores each core's busy% over the last window and +/// moves each cluster's OPP from its busiest core: any saturated core jumps the +/// cluster to its top OPP, and the cluster sheds a step only when all its cores +/// are near-idle. Applies via SCMI+PMIC. Pure w.r.t. the task runtime — it +/// neither sleeps nor spawns — so this crate needs no dependency on +/// ax-task/ax-hal (which would be a cyclic dep through axplat-dyn). +/// +/// `busy[i]` is CPU `i`'s counter; indices past the slice, or offline CPUs whose +/// counter never advances, simply read as idle — conservative (never over-scales). +pub fn governor_poll(busy: &[u64]) { + if !governor_wanted() { + return; + } + + // Measure the ACTUAL window this poll covers, in scheduler ticks, from the + // monotonic clock — not the nominal `GOV_PERIOD_MS`. If the governor task woke + // late (the common case under load), the real window is longer than nominal and + // more busy ticks accumulated; dividing that larger delta by the nominal window + // would inflate busy% and spuriously jump a moderate cluster to its top OPP. + // Floor at 1 tick so a back-to-back poll (< 10 ms apart) can't divide by zero. + let now_nanos = axklib::time::monotonic_nanos(); + let last_nanos = LAST_POLL_NANOS.swap(now_nanos, Ordering::Relaxed); + let window_ticks = (now_nanos.saturating_sub(last_nanos) / NANOS_PER_TICK).max(1); + + // First call only establishes the baseline (see `PRIMED`); still walk every + // cluster below so all `LAST_BUSY` entries are seeded, but make no OPP change. + // (On this priming poll `last_nanos == 0` makes `window_ticks` huge, so every + // busy% reads ~0 — irrelevant, since the priming poll holds regardless.) + let priming = !PRIMED.swap(true, Ordering::Relaxed); + + for (ci, &cluster) in [Cluster::A55, Cluster::Big0, Cluster::Big1] + .iter() + .enumerate() + { + // Score each core in the cluster individually this window. A cluster + // shares ONE clock, so a single saturated core is reason to raise the + // whole cluster — this matches Linux schedutil/ondemand, which drive a + // frequency domain from its busiest CPU. Averaging instead (an earlier + // bug) buried one CPU-bound thread among its idle siblings: a single + // thread on the 2-core A76 pair only reads 50%, below the up-threshold, + // so the cluster never boosted. Each CPU belongs to exactly one cluster, + // so every LAST_BUSY entry is refreshed exactly once per poll. + let mut peak_pct = 0u64; // busiest core, for the log line + // RK3588 clusters have at most 4 cores (A55 cpu0-3; the big pairs have 2). + let mut core_pcts = [0u64; 4]; + let mut n = 0usize; + for cpu in cluster.cpus() { + let now = busy.get(cpu).copied().unwrap_or(0); + let last = LAST_BUSY[cpu].swap(now, Ordering::Relaxed); + // Per-core busy% = busy_ticks / actual window_ticks (one core), clamped. + let pct = ((now.saturating_sub(last) * 100) / window_ticks).min(100); + if pct > peak_pct { + peak_pct = pct; + } + if let Some(slot) = core_pcts.get_mut(n) { + *slot = pct; + } + n += 1; + } + if n == 0 { + continue; + } + + let opps = cluster.opps(); + let cur = IDX[ci].load(Ordering::Relaxed); + // Pure up/down/hold/prime decision (host-tested); the priming poll only + // seeds the baseline above and holds here. + let new = next_opp_idx( + &core_pcts[..n.min(core_pcts.len())], + cur, + opps.len(), + priming, + ); + + if new != cur { + // Only commit IDX (the software record of the last CONFIRMED OPP) + // when both the voltage write and the SCMI clock set are verified + // successful. On failure, IDX is left at `cur` — the hardware is + // always left in a safe (never-undervolted, see `apply_opp`) state + // for that index, so the next poll retries the same climb/descent + // from a known-good starting point rather than silently pretending + // the change took effect. + if apply_opp(cluster, opps[new], new > cur) { + IDX[ci].store(new, Ordering::Relaxed); + info!( + "gov: {} peak={}% opp {}->{} = {} MHz @ {} mV", + cluster.name(), + peak_pct, + cur, + new, + opps[new].mhz, + opps[new].uv / 1_000 + ); + } else { + warn!( + "gov: {} peak={}% opp {}->{} FAILED to apply; staying at {} ({} MHz @ {} mV)", + cluster.name(), + peak_pct, + cur, + new, + cur, + opps[cur].mhz, + opps[cur].uv / 1_000 + ); + } + } + } +} + +// =========================================================================== +// OPP calibration sweep (gated, one-shot) +// =========================================================================== +// +// The PVTPLL clock is voltage-coupled, so the delivered frequency for a given +// SCMI ring drifts with rail voltage, and the drift grows at the higher OPPs +// (ring=1608 @ 762.5 mV measured near ~1733 MHz on-board). To use the +// 1416/1608/1800 rungs SAFELY we must know the *actual* delivered frequency at +// each (ring, voltage); this sweep measures it directly via the PMU cycle +// counter. It runs once, gated by `CALIBRATE`, from early `init()` (before the +// console tty handoff) so its `CAL` log lines reach the serial console — the +// governor's own transition logs do not, because they fire post-handoff. + +/// One-shot gate: when true, `init()` runs [`calibrate_cluster`] per cluster +/// (governor NOT spawned) and the board logs a `CAL` grid; leave false for +/// production. Requires the PMU cycle counter enabled at boot (axcpu `init_trap`). +const CALIBRATE: bool = false; + +/// Sweep points `(rail_uV, ring_kHz)`. Round 1 showed the delivered frequency is +/// dominated by voltage and that at any DT (ring=F, V_nom(F)) pair the delivery +/// *over*-shoots F (undervolt). The safe lever is instead a FIXED low ring with a +/// rising voltage: the delivered freq climbs but stays over-volted. So round 2 +/// maps ring 1200 across the full voltage range (plus two higher-ring cross-checks +/// to see where the ring stops mattering). Every point keeps V >= V_nom(ring), so +/// no measured point undervolts a live core; list is voltage-non-decreasing. +const CAL_A76: &[(u32, u32)] = &[ + (675_000, 1_200_000), + (725_000, 1_200_000), + (762_500, 1_200_000), + (800_000, 1_200_000), + (850_000, 1_200_000), + (925_000, 1_200_000), + (850_000, 1_416_000), // cross-check: does a higher ring beat ring 1200 at 850? + (925_000, 1_608_000), // cross-check at 925 +]; + +/// A55: fixed ring 1008 across the voltage range (RK806 force-write caps at 950 mV). +const CAL_A55: &[(u32, u32)] = &[ + (675_000, 1_008_000), + (712_500, 1_008_000), + (762_500, 1_008_000), + (800_000, 1_008_000), + (850_000, 1_008_000), + (950_000, 1_008_000), + (850_000, 1_416_000), // cross-check + (950_000, 1_608_000), // cross-check +]; + +/// Whether to run the calibration sweep this boot (compile gate + PMIC armed). +pub fn calibrate_wanted() -> bool { + CALIBRATE && GOV_READY.load(Ordering::Acquire) +} + +// The calibration sweep is board-only (gated off by `CALIBRATE = false` in +// production), but `rk3588-cpufreq` is a public feature and must still COMPILE +// on a non-aarch64 host (e.g. a plain `cargo build`/`cargo test` on the CI +// host). These four leaf reads are the only AArch64 inline asm in this module; +// everything above them (`measure_mhz`, `cal_delay_ms`, `calibrate_cluster`) +// is arch-generic and calls only these, so gating just the leaves keeps the +// aarch64/board behavior byte-for-byte identical while giving every other +// target a harmless stub (0 never causes a hang: `cal_delay_ms`'s `frq.max(1)` +// and `measure_mhz`'s zero-length window both degenerate to an immediate +// return rather than spinning). +#[cfg(target_arch = "aarch64")] +#[inline] +fn rd_pmccntr() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, pmccntr_el0", out(reg) v) }; + v +} +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn rd_pmccntr() -> u64 { + 0 +} + +#[cfg(target_arch = "aarch64")] +#[inline] +fn rd_cntvct() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, cntvct_el0", out(reg) v) }; + v +} +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn rd_cntvct() -> u64 { + 0 +} + +#[cfg(target_arch = "aarch64")] +#[inline] +fn rd_cntfrq() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) v) }; + v +} +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn rd_cntfrq() -> u64 { + 0 +} + +#[cfg(target_arch = "aarch64")] +#[inline] +fn rd_mpidr() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) v) }; + v +} +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn rd_mpidr() -> u64 { + 0 +} + +/// Busy-wait `ms` milliseconds against the fixed-rate `CNTVCT` clock. +fn cal_delay_ms(ms: u64) { + let frq = rd_cntfrq().max(1); + let start = rd_cntvct(); + let ticks = frq * ms / 1000; + while rd_cntvct().wrapping_sub(start) < ticks { + core::hint::spin_loop(); + } +} + +/// Measure the CURRENT core's frequency (MHz) by counting CPU cycles +/// (`PMCCNTR_EL0`) over a fixed ~60 ms window timed by `CNTVCT_EL0`. The counter +/// is 32-bit (no `PMCR_EL0.LC`), which cannot wrap over this window. +fn measure_mhz() -> u32 { + let frq = rd_cntfrq().max(1); + let window = frq * 60 / 1000; // 60 ms in cntvct ticks + let t0 = rd_cntvct(); + let c0 = rd_pmccntr() as u32; + while rd_cntvct().wrapping_sub(t0) < window { + core::hint::spin_loop(); + } + let t1 = rd_cntvct(); + let c1 = rd_pmccntr() as u32; + let dvct = t1.wrapping_sub(t0).max(1); + let dcyc = c1.wrapping_sub(c0) as u64; // 32-bit wrap-safe + ((dcyc * frq) / (dvct * 1_000_000)) as u32 +} + +/// Run the (voltage x ring) calibration sweep for one cluster ON THE CURRENT +/// CORE, logging the delivered frequency at each point. `cluster_idx`: 0=A55, +/// 1=A76 big0, 2=A76 big1. The caller must have pinned this task onto a core in +/// the target cluster (the logged `MPIDR` aff1 lets you confirm it). Restores a +/// safe boot OPP for the cluster on exit. PMIC access here is pure polling, so it +/// is safe on a non-boot core. +pub fn calibrate_cluster(cluster_idx: usize, intended_cpu: usize) { + let (cluster, points, restore_khz) = match cluster_idx { + 0 => (Cluster::A55, CAL_A55, 1_008_000u32), + 1 => (Cluster::Big0, CAL_A76, 1_200_000u32), + _ => (Cluster::Big1, CAL_A76, 1_200_000u32), + }; + let mpidr = rd_mpidr(); + info!( + "CAL begin cl={} cpu={} mpidr_aff1={} aff0={}", + cluster.name(), + intended_cpu, + (mpidr >> 8) & 0xff, + mpidr & 0xff + ); + let phandle = Phandle::from(0u32); + for &(uv, khz) in points { + // Voltage first (points are voltage-non-decreasing, so this only ever + // over-volts the current ring = safe), then the SCMI ring. + let vok = cluster.set_voltage(uv); + scmi::set_clock_rate(phandle, cluster.clock_id(), khz as u64 * 1_000); + cal_delay_ms(25); + let f = measure_mhz(); + info!( + "CAL cl={} volt={}uV ring={}MHz volt_ok={} => {}MHz", + cluster.name(), + uv, + khz / 1_000, + vok, + f + ); + } + // Restore: ring down first, then voltage down (safe order). + scmi::set_clock_rate(phandle, cluster.clock_id(), restore_khz as u64 * 1_000); + cluster.set_voltage(675_000); + info!( + "CAL end cl={} restored {}MHz@675mV", + cluster.name(), + restore_khz / 1_000 + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + // ----------------------------------------------------------------------- + // OPP transactional apply: ordering + commit-only-on-full-success + // + // These pin the two prior state-machine fixes: `apply_opp` orders the levers + // so the voltage-coupled clock never overshoots, and commits (returns `true`, + // which is what lets `governor_poll` advance `IDX`) ONLY when both the voltage + // and SCMI steps succeed. They exercise the exact pure helpers `apply_opp` + // uses, so a partial-failure regression (advancing past a step that failed) + // would fail here. + // ----------------------------------------------------------------------- + + /// Run `run_apply_steps` with injected per-step outcomes (keyed by execution + /// order), recording which steps actually ran — exactly how `apply_opp` drives + /// the two levers, minus the hardware. + fn run_recording(order: [ApplyStep; 2], outcomes: [bool; 2]) -> (bool, [Option; 2]) { + let mut ran: [Option; 2] = [None, None]; + let mut i = 0usize; + let committed = run_apply_steps(order, |step| { + ran[i] = Some(step); + let ok = outcomes[i]; + i += 1; + ok + }); + (committed, ran) + } + + #[test] + fn apply_step_order_up_is_voltage_then_clock_down_is_clock_then_voltage() { + assert_eq!( + apply_step_order(true), + [ApplyStep::Voltage, ApplyStep::Clock] + ); + assert_eq!( + apply_step_order(false), + [ApplyStep::Clock, ApplyStep::Voltage] + ); + } + + #[test] + fn apply_commits_only_when_both_steps_succeed() { + let (committed, ran) = run_recording(apply_step_order(true), [true, true]); + assert!(committed, "both steps confirmed must commit the OPP"); + assert_eq!(ran, [Some(ApplyStep::Voltage), Some(ApplyStep::Clock)]); + } + + #[test] + fn apply_first_step_failure_skips_second_and_does_not_commit() { + // UPSHIFT with the voltage (first) step failing: the SCMI ring must NOT be + // touched, and the OPP must not commit. + let (committed, ran) = run_recording(apply_step_order(true), [false, true]); + assert!(!committed, "a failed first step must not commit the OPP"); + assert_eq!( + ran, + [Some(ApplyStep::Voltage), None], + "the second lever must be skipped after the first step fails" + ); + } + + #[test] + fn apply_second_step_failure_does_not_commit() { + // DOWNSHIFT with the voltage (second) step failing after the ring lowered: + // both steps ran, but the OPP must not commit (hardware is left over-volted + // for the new lower clock — safe, never under-volted). + let (committed, ran) = run_recording(apply_step_order(false), [true, false]); + assert!(!committed, "a failed second step must not commit the OPP"); + assert_eq!(ran, [Some(ApplyStep::Clock), Some(ApplyStep::Voltage)]); + } + + // ----------------------------------------------------------------------- + // Ondemand governor policy (pure): up / down / hold / prime / floor + // ----------------------------------------------------------------------- + + #[test] + fn governor_saturated_core_jumps_to_top_opp() { + // A single saturated core in a 2-core big pair drives the whole cluster to + // its top OPP (fast attack; the up-threshold is inclusive). + let top = A76_OPPS.len() - 1; + assert_eq!( + next_opp_idx(&[100, 0], BOOT_OPP_IDX, A76_OPPS.len(), false), + top + ); + assert_eq!( + next_opp_idx(&[UP_THRESHOLD_PCT, 0], 0, A76_OPPS.len(), false), + top + ); + } + + #[test] + fn governor_all_idle_sheds_one_step() { + assert_eq!( + next_opp_idx(&[0, 0], BOOT_OPP_IDX, A76_OPPS.len(), false), + BOOT_OPP_IDX - 1 + ); + // Every A55 core just under the down-threshold sheds exactly one step + // (from the top of the ring-only A55 ladder down one rung). + assert_eq!( + next_opp_idx( + &[DOWN_THRESHOLD_PCT - 1; 4], + A55_OPPS.len() - 1, + A55_OPPS.len(), + false + ), + A55_OPPS.len() - 2 + ); + } + + #[test] + fn governor_does_not_shed_below_bottom_opp() { + assert_eq!(next_opp_idx(&[0, 0], 0, A76_OPPS.len(), false), 0); + } + + #[test] + fn governor_holds_on_moderate_load() { + // No core saturated and not every core idle: hold. + let mid = (DOWN_THRESHOLD_PCT + UP_THRESHOLD_PCT) / 2; + assert_eq!( + next_opp_idx(&[mid, mid], BOOT_OPP_IDX, A76_OPPS.len(), false), + BOOT_OPP_IDX + ); + // One busy-but-not-saturated core keeps the cluster put (down-threshold is + // exclusive at the top, up-threshold not yet reached). + assert_eq!( + next_opp_idx( + &[UP_THRESHOLD_PCT - 1, 0], + BOOT_OPP_IDX, + A76_OPPS.len(), + false + ), + BOOT_OPP_IDX + ); + } + + #[test] + fn governor_priming_and_degenerate_inputs_make_no_change() { + // The first (priming) poll only seeds the busy baseline; even a saturated + // reading must not move the OPP that window. + assert_eq!( + next_opp_idx(&[100, 100], BOOT_OPP_IDX, A76_OPPS.len(), true), + BOOT_OPP_IDX + ); + // An empty cluster or an empty ladder also holds. + assert_eq!( + next_opp_idx(&[], BOOT_OPP_IDX, A76_OPPS.len(), false), + BOOT_OPP_IDX + ); + assert_eq!(next_opp_idx(&[100], BOOT_OPP_IDX, 0, false), BOOT_OPP_IDX); + } + + // ----------------------------------------------------------------------- + // OPP-table invariants + // ----------------------------------------------------------------------- + + #[test] + fn opp_table_voltages_within_pmic_envelope() { + // Every ladder rung must be a voltage its rail's PMIC layer will actually + // accept, or the governor's transition would silently fail. The A76 rails + // (RK8602/03 over I2C) accept [VDD_FLOOR_UV, VDD_CEIL_UV] = [675_000, + // 1_000_000] uV (see pmic_i2c); the A55 rail (RK806 force-write over SPI) + // accepts [675_000, 950_000] uV (see pmic_spi::force_write_dcdc2). Both + // bounds are inclusive. (This complements pmic_i2c's own envelope test.) + for opp in A76_OPPS { + assert!( + (675_000..=1_000_000).contains(&opp.uv), + "A76 OPP {} mV outside the I2C PMIC envelope", + opp.uv / 1_000 + ); + } + for opp in A55_OPPS { + assert!( + (675_000..=950_000).contains(&opp.uv), + "A55 OPP {} mV outside the SPI PMIC envelope", + opp.uv / 1_000 + ); + } + } + + #[test] + fn a55_ladder_is_ring_only_on_the_boot_rail() { + // The A55 RK806 rail cannot be read back (MISO scope-wall), so the little + // cluster forgoes the voltage lever: every A55 rung must sit on the single + // boot-confirmed 675 mV rail (see `Cluster::voltage_managed`). This locks in + // the cluster split — a future rung that raised the A55 voltage on an + // unconfirmable SPI write would fail here. + assert!( + !Cluster::A55.voltage_managed(), + "A55 must stay voltage-unmanaged (ring-only) until its rail reads back" + ); + for opp in A55_OPPS { + assert_eq!( + opp.uv, 675_000, + "A55 rung {} MHz must stay on the 675 mV boot rail (ring-only)", + opp.mhz + ); + } + // The A76 clusters keep the full voltage lever. + assert!(Cluster::Big0.voltage_managed()); + assert!(Cluster::Big1.voltage_managed()); + } + + #[test] + fn opp_table_rings_within_boot_safe_ceiling() { + // The ladders never ring a cluster above its boot-safe ceiling: the top + // rungs hold the boot ring and raise only voltage. Guards against a future + // rung that would ring past the ceiling the probe verified. + for opp in A76_OPPS { + assert!( + opp.ring_khz as u64 * 1_000 <= A76_MAX_HZ, + "A76 ring {} kHz above the boot-safe ceiling", + opp.ring_khz + ); + } + for opp in A55_OPPS { + assert!( + opp.ring_khz as u64 * 1_000 <= A55_MAX_HZ, + "A55 ring {} kHz above the boot-safe ceiling", + opp.ring_khz + ); + } + } +} diff --git a/drivers/ax-driver/src/soc/rockchip/mod.rs b/drivers/ax-driver/src/soc/rockchip/mod.rs index a90783299d..f31b7c2003 100644 --- a/drivers/ax-driver/src/soc/rockchip/mod.rs +++ b/drivers/ax-driver/src/soc/rockchip/mod.rs @@ -12,6 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(feature = "rk3588-cpufreq")] +pub(crate) mod cpufreq; + +#[cfg(feature = "rk3588-cpufreq")] +mod pmic_i2c; + +#[cfg(feature = "rk3588-cpufreq")] +mod pmic_spi; + #[cfg(feature = "rockchip-soc")] mod cru; diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs new file mode 100644 index 0000000000..2dcb88d3f1 --- /dev/null +++ b/drivers/ax-driver/src/soc/rockchip/pmic_i2c.rs @@ -0,0 +1,803 @@ +// Copyright 2025 The Axvisor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RK3588 A76 CPU-rail regulator lever (DVFS Phase 2) — RK8602/RK8603 over the +//! `rk3x_i2c` PMIC bus. +//! +//! This is the **voltage half** of the RK3588 CPU-DVFS fix. Phase 1a raises the +//! A76 cluster clocks via SCMI/BL31, but RK3588's CPU clock is voltage-coupled, +//! so the clock overshoots the request while the rail stays at its 800 mV boot +//! value. Lowering each A76 rail to its OPP-nominal 675 mV pulls the coupled +//! clock down to the exact requested rate. This module provides the read/write +//! API; `cpufreq.rs` owns when it is called. +//! +//! # Components +//! +//! * A minimal **polling** `rk3x_i2c` master for `i2c@fd880000` (RK3588 "bus0", +//! the PMIC bus). No IRQs: it polls the raw IPD status. u-boot has already +//! clocked and used this controller (it programs these same PMICs at boot), +//! so [`Rk3xI2c::init_controller`] preserves u-boot's `CLKDIV` (a proven-good +//! SCL rate) and only clears state. Register map + transaction flow modelled +//! on mainline u-boot `drivers/i2c/rk_i2c.c` and Linux +//! `drivers/i2c/busses/i2c-rk3x.c`. +//! * A **RK8602/RK8603** regulator API (`get_uv` / `set_uv` / `set_uv_stepped`) +//! for the two A76 rails (RK8602 @ `0x42` = big0/cpu4-5, RK8603 @ `0x43` = +//! big1/cpu6-7). +//! +//! # RK8602/RK8603 VSEL register (finding) +//! +//! Linux handles RK8602/RK8603 in `drivers/regulator/fan53555.c` +//! (`RK8602_VENDOR_ROCKCHIP`). It defines two voltage registers, +//! `RK8602_VSEL0 = 0x06` and `RK8602_VSEL1 = 0x07`, and selects the **active +//! runtime** register (`di->vol_reg`) from the `sleep_vsel_id` (DT +//! `fcs,suspend-voltage-selector`): `sleep_vsel_id == 1` makes VSEL1 (`0x07`) +//! the *sleep* register and **VSEL0 (`0x06`) the active** one. The Orange Pi +//! 5 Plus RK8602/8603 nodes use `fcs,suspend-voltage-selector = <1>`, so the +//! board's live VSEL is `0x06` — matching the board-confirmed boot value +//! (`0x06 == 0x30 == 800 mV`). We therefore write/read **reg `0x06`**. +//! +//! The voltage code is a full 8-bit field (`RK8602_NVOLTAGES = 160`, +//! `vsel_mask = 0xff`) — no enable/mode bit shares reg `0x06` (the buck-enable +//! `VSEL_BUCK_EN` bit lives in the legacy FAN53555 registers `0x00/0x01`, left +//! untouched — u-boot already enabled the rail). Encoding matches the board: +//! `di->vsel_min = 500000`, `di->vsel_step = 6250` → `V = 500000 + code·6250` +//! µV, i.e. `0x1c → 675 mV`, `0x30 → 800 mV`. + +use core::{ + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +// PMIC access is slow: a register read/write busy-waits on the I2C hardware +// (`wait_ipd`, ~100 ms budget, longer if the bus is dead). Serialisation must NOT +// disable preemption across that poll, or the `cpufreq` governor task would stall +// task-switching on its CPU for the whole transaction (a scheduling-starvation +// risk when the bus misbehaves). So the slow chip transactions are guarded by a +// non-preempt-disabling bus-ownership flag (`BUS_BUSY` / [`BusGuard`]) and the +// `SpinNoPreempt` `Mutex` below is used ONLY to store the controller handle +// (`init`) or clone it out (`BusGuard::claim`) — a microsecond-scale window that +// never spans a poll. Ownership is single-caller by construction (one governor +// task; `init` runs before it), so `BusGuard::claim` takes ownership without +// blocking and bails on the (never-expected) contended case rather than spinning. +// `SpinNoPreempt` (not `SpinNoIrq`) keeps local IRQs enabled; the lock is never +// taken from an interrupt handler, so no re-entrant self-deadlock is possible. +use ax_kspin::SpinNoPreempt as Mutex; +use log::{info, warn}; +use mmio_api::{MmioAddr, MmioRaw}; +use rdif_pinctrl::PinctrlDevice; + +use crate::mmio::iomap; + +// --------------------------------------------------------------------------- +// rk3x_i2c register map (offsets from the controller base) +// --------------------------------------------------------------------------- + +const REG_CON: usize = 0x00; // control +const REG_CLKDIV: usize = 0x04; // clock divider +const REG_MRXADDR: usize = 0x08; // master-rx slave address +const REG_MRXRADDR: usize = 0x0c; // master-rx slave register address +const REG_MTXCNT: usize = 0x10; // master-tx byte count +const REG_MRXCNT: usize = 0x14; // master-rx byte count +const REG_IEN: usize = 0x18; // interrupt enable +const REG_IPD: usize = 0x1c; // interrupt pending (raw status we poll) +const TXDATA_BASE: usize = 0x100; // tx FIFO word 0 +const RXDATA_BASE: usize = 0x200; // rx FIFO word 0 + +// REG_CON bits +const CON_EN: u32 = 1 << 0; +const CON_START: u32 = 1 << 3; +const CON_STOP: u32 = 1 << 4; +const CON_LASTACK: u32 = 1 << 5; // NACK after the last received byte + +// REG_CON transfer modes (shifted into bits [2:1] by `con_mod`) +const MODE_TX: u32 = 0; // write: address + data from tx FIFO +const MODE_TRX: u32 = 1; // combined: write reg address, then read + +const fn con_mod(mode: u32) -> u32 { + mode << 1 +} + +// REG_MRXADDR/REG_MRXRADDR: BIT(24 + n) marks address byte n valid. We only +// ever send one address byte and one register byte, so byte 0 valid = BIT(24). +const MRXADDR_VALID0: u32 = 1 << 24; + +// REG_IPD/REG_IEN interrupt bits +const INT_MBTF: u32 = 1 << 2; // master byte-transmit finished +const INT_MBRF: u32 = 1 << 3; // master byte-receive finished +const INT_START: u32 = 1 << 4; // START generated +const INT_STOP: u32 = 1 << 5; // STOP generated +const INT_NAKRCV: u32 = 1 << 6; // NACK received +const INT_ALL: u32 = 0x7f; // all pending bits (write-1-to-clear) + +/// RK3588 PMIC I2C controller ("bus0", `i2c@fd880000`). +const RK3588_I2C0_BASE: usize = 0xfd88_0000; +const RK3588_I2C0_SIZE: usize = 0x1000; + +// --- i2c0 clock ungate (PMU CRU) -------------------------------------------- +// +// i2c0 lives in the PMU CRU (`pmucru: clock-controller@fd7f0000` = main CRU +// 0xfd7c0000 + 0x30000). u-boot programmed the PMICs over i2c0 at boot but +// StarryOS leaves **CLK_I2C0 gated**: PCLK is still on (registers readable, e.g. +// CLKDIV reads back), yet the controller cannot generate SCL, so any START +// times out. `init` ungates it before the first transaction. +// +// Register/bit ground truth (cross-checked, identical in both): +// * mainline `drivers/clk/rockchip/clk-rk3588.c`: +// GATE(PCLK_I2C0, ..., RK3588_PMU_CLKGATE_CON(2), 1) +// COMPOSITE_NODIV(CLK_I2C0, ..., RK3588_PMU_CLKGATE_CON(2), 2) +// * `rockchip-soc` CRU crate gate.rs: `PCLK_I2C0 => (2, 1)`, `CLK_I2C0 => (2, 2)`. +// * PMU_CLKGATE_CON(x) = 0x800 + x*4 within the PMU CRU → CON(2) = 0x808. +/// PMU CRU physical base (`clock-controller@fd7f0000`). +const RK3588_PMU_CRU_BASE: usize = 0xfd7f_0000; +const RK3588_PMU_CRU_SIZE: usize = 0x1000; +/// `PMU_CLKGATE_CON(2)` offset within the PMU CRU (holds the i2c0 gates). +const PMU_CLKGATE_CON2: usize = 0x800 + 2 * 4; +/// Gate bits to clear: PCLK_I2C0 (bit 1) + CLK_I2C0 (bit 2). +const I2C0_GATE_BITS: u32 = (1 << 1) | (1 << 2); + +// i2c0 soft-reset is in the same PMU CRU. Ungating the clock alone was not +// enough on-board (registers readable, but START still timed out): the i2c0 +// core is held in **soft-reset**. SRST_P_I2C0 (PCLK reset) = bit 1 and +// SRST_I2C0 (functional/core reset) = bit 2 of PMU_SOFTRST_CON(2), from mainline +// `drivers/clk/rockchip/rst-rk3588.c:776-777` +// (`RK3588_PMU1CRU_RESET_OFFSET(SRST_P_I2C0, 2, 1)` / `(SRST_I2C0, 2, 2)`) — +// same bit layout as the gate. The functional reset (bit 2) matches the +// symptom: PCLK regs read (SRST_P deasserted) but the SCL state machine is held. +// We only DE-ASSERT (write 0, masked), never assert, so u-boot's preserved +// CLKDIV in the PCLK domain is not cleared. +/// `PMU_SOFTRST_CON(2)` offset within the PMU CRU (0xa00 + 2*4). +const PMU_SOFTRST_CON2: usize = 0xa00 + 2 * 4; +/// Reset bits to release: SRST_P_I2C0 (bit 1) + SRST_I2C0 (bit 2). +const I2C0_RESET_BITS: u32 = (1 << 1) | (1 << 2); + +/// Poll budget for a single IPD event, at 1 µs granularity — ~100 ms, matching +/// u-boot's `I2C_TIMEOUT_MS`. A PMIC register access completes in tens of µs; +/// this only bounds a wedged bus. +const I2C_POLL_MAX: u32 = 100_000; + +// --------------------------------------------------------------------------- +// RK8602/RK8603 regulator constants +// --------------------------------------------------------------------------- + +/// RK8602 @ I2C `0x42` — A76 big0 rail (`vdd_cpu_big0_s0`, cpu4-5). +pub const RK8602_BIG0_ADDR: u8 = 0x42; +/// RK8603 @ I2C `0x43` — A76 big1 rail (`vdd_cpu_big1_s0`, cpu6-7). +pub const RK8603_BIG1_ADDR: u8 = 0x43; + +/// Active runtime VSEL register on this board (RK8602_VSEL0). See module docs. +const VSEL_REG: u8 = 0x06; +/// 8-bit voltage-code mask (`RK8602_NVOLTAGES = 160`, `vsel_mask = 0xff`). +const VSEL_MASK: u8 = 0xff; +/// Voltage encoding: `V = VSEL_BASE_UV + code · VSEL_STEP_UV` (µV). +const VSEL_BASE_UV: u32 = 500_000; +const VSEL_STEP_UV: u32 = 6_250; + +/// Safety envelope for the A76 rails. Floor = the lowest OPP nominal (675 mV); +/// ceiling = the top OPP voltage (1.0 V @ 2256 MHz). `set_uv*` refuse anything +/// outside. The governor only ever passes OPP-matched voltages (each a +/// Linux-proven freq/voltage pair), so any accepted value is proven-safe; the +/// clamp is defense-in-depth against a bad caller. (Phase-1a only sets 675 mV.) +const VDD_FLOOR_UV: u32 = 675_000; +const VDD_CEIL_UV: u32 = 1_000_000; + +/// Maximum voltage change per stepped-lowering step (≤ 4 LSB = 25 mV), so the +/// voltage-coupled clock tracks the rail down gradually. +const STEP_MAX_UV: u32 = 25_000; +/// Settle delay after each step, giving the rail time to slew before the next. +const SETTLE_US: u64 = 500; + +// --------------------------------------------------------------------------- +// Voltage <-> VSEL code (pure, host-testable) +// --------------------------------------------------------------------------- + +/// Decode an 8-bit VSEL code to microvolts. +fn vsel_to_uv(vsel: u8) -> u32 { + VSEL_BASE_UV + (vsel as u32) * VSEL_STEP_UV +} + +/// Encode microvolts to a VSEL code, requiring an **exact** round-trip. Returns +/// `None` for values below the base, above the 8-bit range, or that do not land +/// exactly on a 6.25 mV step (so we never program an approximated voltage). +fn uv_to_vsel(uv: u32) -> Option { + if uv < VSEL_BASE_UV { + return None; + } + let code = (uv - VSEL_BASE_UV) / VSEL_STEP_UV; + if code > VSEL_MASK as u32 { + return None; + } + let vsel = code as u8; + (vsel_to_uv(vsel) == uv).then_some(vsel) +} + +/// Whether `uv` is inside the Phase-2 down-only safety envelope. +fn in_envelope(uv: u32) -> bool { + (VDD_FLOOR_UV..=VDD_CEIL_UV).contains(&uv) +} + +// --------------------------------------------------------------------------- +// rk3x_i2c polling master +// --------------------------------------------------------------------------- + +/// Outcome of waiting on an I2C completion event. +enum I2cErr { + Nak, + Timeout, +} + +#[derive(Clone)] +struct Rk3xI2c { + mmio: MmioRaw, +} + +impl Rk3xI2c { + #[inline] + fn r(&self, off: usize) -> u32 { + self.mmio.read::(off) + } + + #[inline] + fn w(&self, off: usize, val: u32) { + self.mmio.write::(off, val); + } + + /// Prepare the controller for polled use. u-boot has already clocked and + /// used this bus, so its `CLKDIV` is a proven-good SCL rate; preserve it and + /// only fall back to a conservative ~100 kHz divider (assuming RK3588's + /// 200 MHz i2c function clock) if it is somehow unset. Then park the + /// controller disabled with interrupts off and pending status cleared. + fn init_controller(&self) { + if self.r(REG_CLKDIV) == 0 { + // div = ceil(200 MHz / (100 kHz · 8)) - 2 = 248; split high/low. + let (divl, divh) = (124u32, 124u32); + self.w(REG_CLKDIV, (divh << 16) | divl); + warn!("pmic_i2c: CLKDIV was 0; set conservative ~100 kHz (divl=divh=124 @200 MHz)"); + } + self.w(REG_CON, 0); + self.w(REG_IEN, 0); + self.w(REG_IPD, INT_ALL); + info!( + "pmic_i2c: rk3x_i2c@{:#x} ready (clkdiv={:#x})", + RK3588_I2C0_BASE, + self.r(REG_CLKDIV) + ); + } + + /// Poll IPD until any bit in `mask` is set. A received NACK aborts early. + fn wait_ipd(&self, mask: u32) -> Result<(), I2cErr> { + for _ in 0..I2C_POLL_MAX { + let ipd = self.r(REG_IPD); + if ipd & INT_NAKRCV != 0 { + self.w(REG_IPD, INT_NAKRCV); + return Err(I2cErr::Nak); + } + if ipd & mask != 0 { + self.w(REG_IPD, mask); + return Ok(()); + } + axklib::time::busy_wait(Duration::from_micros(1)); + } + Err(I2cErr::Timeout) + } + + fn send_start(&self) -> Result<(), I2cErr> { + self.w(REG_IPD, INT_ALL); + self.w(REG_CON, CON_EN | CON_START); + self.w(REG_IEN, INT_START); + let res = self.wait_ipd(INT_START); + if res.is_err() { + // Board diagnostic: if START never completes, dump the controller + // state. A live controller sets INT_START(bit4) in IPD; CON reads + // back EN|START; CLKDIV should hold the (u-boot) divisor. All-zero + // or stuck values point at a still-gated clock or held reset. + warn!( + "pmic_i2c: START did not complete: CON={:#010x} IPD={:#010x} CLKDIV={:#010x} \ + IEN={:#010x}", + self.r(REG_CON), + self.r(REG_IPD), + self.r(REG_CLKDIV), + self.r(REG_IEN) + ); + } + res + } + + fn send_stop(&self) -> Result<(), I2cErr> { + self.w(REG_IPD, INT_ALL); + self.w(REG_CON, CON_EN | CON_STOP); + self.w(REG_IEN, INT_STOP); + self.wait_ipd(INT_STOP) + } + + #[inline] + fn disable(&self) { + self.w(REG_CON, 0); + } + + /// SMBus-style single-byte register write: `[START][chip.W][reg][val][STOP]`. + fn write_reg(&self, chip: u8, reg: u8, val: u8) -> bool { + if self.send_start().is_err() { + warn!("pmic_i2c: START failed writing chip {chip:#x} reg {reg:#x}"); + self.disable(); + return false; + } + // tx FIFO word 0: byte0 = write address, byte1 = reg, byte2 = data. + let word0 = ((chip as u32) << 1) | ((reg as u32) << 8) | ((val as u32) << 16); + self.w(TXDATA_BASE, word0); + self.w(REG_CON, CON_EN | con_mod(MODE_TX)); + self.w(REG_MTXCNT, 3); + self.w(REG_IEN, INT_MBTF | INT_NAKRCV); + let ok = self.wait_ipd(INT_MBTF); + // STOP before disable (disabling first would emit an illegal START/STOP). + let _ = self.send_stop(); + self.disable(); + match ok { + Ok(()) => true, + Err(I2cErr::Nak) => { + warn!("pmic_i2c: NACK writing chip {chip:#x} reg {reg:#x}"); + false + } + Err(I2cErr::Timeout) => { + warn!("pmic_i2c: timeout writing chip {chip:#x} reg {reg:#x}"); + false + } + } + } + + /// SMBus-style single-byte register read: combined write-address + read. + fn read_reg(&self, chip: u8, reg: u8) -> Option { + if self.send_start().is_err() { + warn!("pmic_i2c: START failed reading chip {chip:#x} reg {reg:#x}"); + self.disable(); + return None; + } + // Read address (byte0 valid) and the register byte to send (byte0 valid). + self.w(REG_MRXADDR, (((chip as u32) << 1) | 1) | MRXADDR_VALID0); + self.w(REG_MRXRADDR, (reg as u32) | MRXADDR_VALID0); + self.w(REG_CON, CON_EN | CON_LASTACK | con_mod(MODE_TRX)); + self.w(REG_MRXCNT, 1); + self.w(REG_IEN, INT_MBRF | INT_NAKRCV); + let res = self.wait_ipd(INT_MBRF); + let val = match res { + Ok(()) => Some((self.r(RXDATA_BASE) & 0xff) as u8), + Err(I2cErr::Nak) => { + warn!("pmic_i2c: NACK reading chip {chip:#x} reg {reg:#x}"); + None + } + Err(I2cErr::Timeout) => { + warn!("pmic_i2c: timeout reading chip {chip:#x} reg {reg:#x}"); + None + } + }; + let _ = self.send_stop(); + self.disable(); + val + } + + /// Write one VSEL code and confirm the read-back matches exactly. + fn set_vsel_verify(&self, chip: u8, vsel: u8) -> bool { + if !self.write_reg(chip, VSEL_REG, vsel) { + return false; + } + match self.read_reg(chip, VSEL_REG) { + Some(rb) if (rb & VSEL_MASK) == vsel => true, + Some(rb) => { + warn!( + "pmic_i2c: chip {chip:#x} VSEL read-back {:#x} != written {vsel:#x}", + rb & VSEL_MASK + ); + false + } + None => { + warn!("pmic_i2c: chip {chip:#x} VSEL read-back failed after write"); + false + } + } + } +} + +// --------------------------------------------------------------------------- +// Public API — single global PMIC bus (mapped once) +// --------------------------------------------------------------------------- + +/// Storage for the initialised controller handle. The `SpinNoPreempt` mutex is +/// held only for the microsecond-scale window that stores it (`init`) or clones +/// it out ([`BusGuard::claim`]) — NEVER across a hardware poll. +static CONTROLLER: Mutex> = Mutex::new(None); + +/// Bus-ownership flag for the slow chip transactions. Unlike a `SpinNoPreempt` +/// guard, holding it does NOT disable preemption, so a `wait_ipd` poll (~100 ms +/// budget, longer if the bus is dead) can be preempted and never starves the +/// scheduler on that CPU — the blocking concern raised in review. +/// +/// Concurrency protocol: the PMIC bus has exactly one steady-state caller, the +/// single `cpufreq` governor task, and `init` runs once at boot before that task +/// exists — so contention does not occur by construction. The flag makes that +/// explicit and safe: [`BusGuard::claim`] tries to take ownership without +/// blocking and *bails* (returns `None` → the caller's safe no-op) if a +/// transaction is somehow already in flight, rather than spinning. A transaction +/// preempted mid-poll simply holds ownership across the gap; the hardware +/// transaction keeps running and resuming the status poll is idempotent. +static BUS_BUSY: AtomicBool = AtomicBool::new(false); + +/// RAII ownership of the PMIC bus for one (possibly multi-step) transaction. +/// Carries a clone of the controller handle so the slow poll runs with no +/// `SpinNoPreempt` guard held. Releases ownership on drop. +struct BusGuard { + i2c: Rk3xI2c, +} + +impl BusGuard { + /// Take exclusive bus ownership without blocking, then clone the controller + /// handle out under a microsecond-scale `CONTROLLER` lock. Returns `None` + /// if the bus is uninitialised or already owned (see the protocol note on + /// [`BUS_BUSY`]); in both cases the caller falls back to its safe no-op. + fn claim() -> Option { + if BUS_BUSY + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + warn!("pmic_i2c: bus already in use; skipping this transaction"); + return None; + } + // Brief preempt-disabled window: just clone the handle out, no poll. + let handle = CONTROLLER.lock().as_ref().cloned(); + match handle { + Some(i2c) => Some(Self { i2c }), + None => { + BUS_BUSY.store(false, Ordering::Release); + None + } + } + } +} + +impl Drop for BusGuard { + fn drop(&mut self) { + BUS_BUSY.store(false, Ordering::Release); + } +} + +impl core::ops::Deref for BusGuard { + type Target = Rk3xI2c; + fn deref(&self) -> &Rk3xI2c { + &self.i2c + } +} + +/// Ungate the i2c0 PCLK + functional clock in the PMU CRU so the controller can +/// generate SCL. Idempotent — the Rockchip CRU gate registers are write-masked +/// (high 16 bits select which bits to write, low 16 are the values), and a `0` +/// in a gate bit means *enabled*, so re-clearing an already-enabled gate is a +/// no-op. Best-effort: on a mapping failure it logs and returns, leaving the +/// clock as-is (the subsequent transaction then times out into a safe no-op). +/// The controller is also released from soft-reset separately, see +/// [`deassert_i2c0_reset`]. +fn ungate_i2c0_clocks() { + let virt = match iomap(RK3588_PMU_CRU_BASE, RK3588_PMU_CRU_SIZE) { + Ok(ptr) => ptr, + Err(err) => { + warn!( + "pmic_i2c: iomap PMU CRU {RK3588_PMU_CRU_BASE:#x} failed: {err:?}; i2c0 clock \ + left as-is" + ); + return; + } + }; + // Clear both gate bits (write mask in the upper half, zeros in the lower). + let val = I2C0_GATE_BITS << 16; + // SAFETY: `virt` maps `RK3588_PMU_CRU_SIZE` bytes at the PMU CRU base; + // `PMU_CLKGATE_CON2` (0x808) is within that window. + let gate = unsafe { virt.as_ptr().add(PMU_CLKGATE_CON2).cast::() }; + unsafe { gate.write_volatile(val) }; + // Let the newly-ungated functional clock start before the first transfer. + axklib::time::busy_wait(Duration::from_micros(5)); + // Read back the (unmasked) gate bits for board diagnostics: 0 == enabled. + let after = unsafe { gate.read_volatile() } & I2C0_GATE_BITS; + info!( + "pmic_i2c: ungated i2c0 clocks (PMU_CLKGATE_CON(2)@{:#x} <- {val:#010x}); gate bits now \ + {after:#x} (0 == enabled)", + RK3588_PMU_CRU_BASE + PMU_CLKGATE_CON2 + ); +} + +/// Release the i2c0 controller from soft-reset in the PMU CRU so its SCL state +/// machine can run. De-assert only (write masked zeros to SRST_P_I2C0 bit 1 + +/// SRST_I2C0 bit 2); we never assert, so the PCLK-domain CLKDIV that u-boot left +/// is preserved. Idempotent (re-clearing an already-released reset is a no-op). +/// Best-effort: a mapping failure is logged and the later transaction just times +/// out into a safe no-op. Reads the bits back for board diagnostics. +fn deassert_i2c0_reset() { + let virt = match iomap(RK3588_PMU_CRU_BASE, RK3588_PMU_CRU_SIZE) { + Ok(ptr) => ptr, + Err(err) => { + warn!( + "pmic_i2c: iomap PMU CRU {RK3588_PMU_CRU_BASE:#x} failed: {err:?}; i2c0 reset \ + left as-is" + ); + return; + } + }; + // Clear both reset bits (write mask in the upper half, zeros in the lower). + let val = I2C0_RESET_BITS << 16; + // SAFETY: `virt` maps `RK3588_PMU_CRU_SIZE` bytes at the PMU CRU base; + // `PMU_SOFTRST_CON2` (0xa08) is within that window. + let softrst = unsafe { virt.as_ptr().add(PMU_SOFTRST_CON2).cast::() }; + unsafe { softrst.write_volatile(val) }; + // Give the core a moment to come out of reset before the first transfer. + axklib::time::busy_wait(Duration::from_micros(10)); + // Read back the (unmasked) reset bits for board diagnostics: 0 == released. + let after = unsafe { softrst.read_volatile() } & I2C0_RESET_BITS; + info!( + "pmic_i2c: de-asserted i2c0 reset (PMU_SOFTRST_CON(2)@{:#x} <- {val:#010x}); reset bits \ + now {after:#x} (0 == released)", + RK3588_PMU_CRU_BASE + PMU_SOFTRST_CON2 + ); +} + +/// Mux the i2c0 SCL/SDA pads to the i2c0 function via the registered Rockchip +/// pinctrl driver. Without this the rk3x master waits forever for a free bus and +/// never generates START (observed on-board: `IPD=0` after ungate + reset): +/// u-boot muxed these pads to i2c0 to program the PMICs, but StarryOS reverts +/// them at boot. The pins are i2c0m2 — `GPIO0_D1` (SCL) / `GPIO0_D2` (SDA) at +/// function 3 — which sit in the GPIO0_D "high group" split PMU2_IOC/BUS_IOC +/// mux path; rather than hand-decode that, we apply the i2c0 node's `pinctrl-0` +/// state through the in-tree pinctrl driver, which already encodes it. This is a +/// mux change only (no PMIC access). Best-effort: any failure is logged and the +/// later transaction times out into a safe no-op. +fn set_i2c0_pinmux() { + let Some(pinctrl) = rdrive::get_one::() else { + warn!("pmic_i2c: PinctrlDevice not registered; cannot mux i2c0 pins (START will fail)"); + return; + }; + let mut pinctrl = match pinctrl.lock() { + Ok(guard) => guard, + Err(err) => { + warn!("pmic_i2c: failed to lock PinctrlDevice: {err}; cannot mux i2c0 pins"); + return; + } + }; + let Some(fdt) = rdrive::with_fdt(Clone::clone) else { + warn!("pmic_i2c: live FDT not found; cannot mux i2c0 pins"); + return; + }; + // The i2c0 controller node (reg base == RK3588_I2C0_BASE) carries + // `pinctrl-0 = <&i2c0m2_xfer>`; several i2c controllers share the + // rk3399-i2c compatible, so match on the reg address. + let node = fdt + .find_compatible(&["rockchip,rk3588-i2c", "rockchip,rk3399-i2c"]) + .into_iter() + .find(|n| { + n.regs() + .into_iter() + .next() + .is_some_and(|r| r.address as usize == RK3588_I2C0_BASE) + }); + let Some(node) = node else { + warn!("pmic_i2c: i2c0 node @ {RK3588_I2C0_BASE:#x} not found in FDT; cannot mux pins"); + return; + }; + match pinctrl.apply_fdt_default_state(&fdt, node.as_node()) { + Ok(()) => info!( + "pmic_i2c: applied i2c0 pinctrl-0 (scl GPIO0_D1 / sda GPIO0_D2 -> func3) via rockchip \ + pinctrl" + ), + Err(err) => warn!("pmic_i2c: failed to apply i2c0 pinctrl-0: {err:?}"), + } +} + +/// Map and initialise the RK3588 PMIC I2C controller. Idempotent; safe to call +/// from a `PostKernel` probe context (MMU up). Returns `false` if the MMIO +/// mapping fails, in which case every `get_uv`/`set_uv*` below is a no-op that +/// returns `None`/`false` and leaves the rails at their (safe) boot voltage. +pub fn init() -> bool { + let mut guard = CONTROLLER.lock(); + if guard.is_some() { + return true; + } + // Bring the i2c0 bus up before any transaction: ungate its clocks, release + // it from soft-reset, and mux its SCL/SDA pads to the i2c0 function (all + // three left closed/reverted at boot; any one makes every START time out). + // All best-effort — a failure just leaves the later transaction to time out + // into a safe no-op. + ungate_i2c0_clocks(); + deassert_i2c0_reset(); + set_i2c0_pinmux(); + let virt = match iomap(RK3588_I2C0_BASE, RK3588_I2C0_SIZE) { + Ok(ptr) => ptr, + Err(err) => { + warn!("pmic_i2c: iomap {RK3588_I2C0_BASE:#x} failed: {err:?}"); + return false; + } + }; + // SAFETY: `virt` is a fresh device mapping of `RK3588_I2C0_BASE` of + // `RK3588_I2C0_SIZE` bytes returned by `iomap`. + let mmio = unsafe { + MmioRaw::new( + MmioAddr::from(RK3588_I2C0_BASE as u64), + virt, + RK3588_I2C0_SIZE, + ) + }; + let i2c = Rk3xI2c { mmio }; + i2c.init_controller(); + *guard = Some(i2c); + true +} + +/// Read a rail's current voltage in microvolts. `chip` is [`RK8602_BIG0_ADDR`] +/// or [`RK8603_BIG1_ADDR`]. `None` if the bus is uninitialised or the read +/// fails. +pub fn get_uv(chip: u8) -> Option { + // Bus ownership (not a preempt-disabling lock) is held across the poll below. + let bus = BusGuard::claim()?; + let vsel = bus.read_reg(chip, VSEL_REG)?; + Some(vsel_to_uv(vsel & VSEL_MASK)) +} + +/// Set a rail directly to `target_uv` (single write), refusing anything outside +/// the Phase-2 envelope `[675_000, 800_000]` µV or that is not an exact 6.25 mV +/// step, and verifying the read-back. Returns `false` on any rejection or +/// mismatch (leaving the rail unchanged/at its last confirmed value). +/// +/// This is the direct setter. Live-core lowering should use +/// [`set_uv_stepped`], which ramps down in small increments. +pub fn set_uv(chip: u8, target_uv: u32) -> bool { + if !in_envelope(target_uv) { + warn!( + "pmic_i2c: refusing chip {chip:#x} set to {target_uv} uV (outside [{VDD_FLOOR_UV}, \ + {VDD_CEIL_UV}])" + ); + return false; + } + let Some(vsel) = uv_to_vsel(target_uv) else { + warn!("pmic_i2c: {target_uv} uV is not an exact VSEL step; refusing"); + return false; + }; + let Some(bus) = BusGuard::claim() else { + warn!("pmic_i2c: not initialised or busy; refusing set"); + return false; + }; + bus.set_vsel_verify(chip, vsel) +} + +/// Safely lower a rail to `target_uv` by stepping **down** in ≤25 mV (4-LSB) +/// increments, verifying the read-back and settling after each step, so the +/// voltage-coupled CPU clock tracks the rail down without a large undervolt +/// transient. This is the path `cpufreq` calls on live cores. +/// +/// Refuses (returns `false`) if `target_uv` is outside the envelope, is not an +/// exact step, or is **above** the rail's current voltage (this path never +/// raises voltage). A no-op success if already at target. Aborts and returns +/// `false` on the first read-back mismatch, leaving the rail at the last +/// verified step. +pub fn set_uv_stepped(chip: u8, target_uv: u32) -> bool { + if !in_envelope(target_uv) { + warn!( + "pmic_i2c: refusing chip {chip:#x} step to {target_uv} uV (outside [{VDD_FLOOR_UV}, \ + {VDD_CEIL_UV}])" + ); + return false; + } + let Some(target_vsel) = uv_to_vsel(target_uv) else { + warn!("pmic_i2c: {target_uv} uV is not an exact VSEL step; refusing"); + return false; + }; + + // One bus claim for the whole stepped transaction (read + the step writes), + // so it stays atomic w.r.t. other callers while never disabling preemption. + let Some(bus) = BusGuard::claim() else { + warn!("pmic_i2c: not initialised or busy; refusing step"); + return false; + }; + + let Some(cur_vsel) = bus.read_reg(chip, VSEL_REG).map(|v| v & VSEL_MASK) else { + warn!("pmic_i2c: chip {chip:#x} current VSEL read failed; refusing step"); + return false; + }; + + if target_vsel > cur_vsel { + warn!( + "pmic_i2c: refusing to step UP chip {chip:#x} {} -> {target_uv} uV (down-only path)", + vsel_to_uv(cur_vsel) + ); + return false; + } + if target_vsel == cur_vsel { + return true; + } + + const STEP_LSB: u8 = (STEP_MAX_UV / VSEL_STEP_UV) as u8; // 25000 / 6250 = 4 + let mut v = cur_vsel; + while v > target_vsel { + let next = v.saturating_sub(STEP_LSB).max(target_vsel); + if !bus.set_vsel_verify(chip, next) { + warn!( + "pmic_i2c: chip {chip:#x} step to VSEL {next:#x} ({} uV) failed; aborting at {} uV", + vsel_to_uv(next), + vsel_to_uv(v) + ); + return false; + } + axklib::time::busy_wait(Duration::from_micros(SETTLE_US)); + v = next; + } + info!( + "pmic_i2c: chip {chip:#x} stepped {} -> {} uV", + vsel_to_uv(cur_vsel), + vsel_to_uv(target_vsel) + ); + true +} + +#[cfg(test)] +mod tests { + use super::*; + + // Board-confirmed VSEL points. + #[test] + fn vsel_decode_matches_board_points() { + assert_eq!(vsel_to_uv(0x1c), 675_000); + assert_eq!(vsel_to_uv(0x20), 700_000); + assert_eq!(vsel_to_uv(0x24), 725_000); + assert_eq!(vsel_to_uv(0x30), 800_000); + assert_eq!(vsel_to_uv(0x50), 1_000_000); + } + + #[test] + fn uv_encode_roundtrips_board_points() { + assert_eq!(uv_to_vsel(675_000), Some(0x1c)); + assert_eq!(uv_to_vsel(700_000), Some(0x20)); + assert_eq!(uv_to_vsel(725_000), Some(0x24)); + assert_eq!(uv_to_vsel(800_000), Some(0x30)); + assert_eq!(uv_to_vsel(1_000_000), Some(0x50)); + } + + #[test] + fn uv_encode_rejects_non_step_and_out_of_range() { + // Not a multiple of 6.25 mV. + assert_eq!(uv_to_vsel(676_000), None); + assert_eq!(uv_to_vsel(675_001), None); + // Below the encoding base. + assert_eq!(uv_to_vsel(499_999), None); + // Above the 8-bit code range (code 255 -> 2.09375 V). + assert_eq!(uv_to_vsel(2_100_000), None); + } + + #[test] + fn envelope_is_675k_to_1000k_inclusive() { + // Safe envelope is [VDD_FLOOR_UV, VDD_CEIL_UV] = [675 mV, 1.0 V]. The + // ceiling was raised from the boot 800 mV to the calibrated 1.0 V + // over-volted safe max needed for the higher A76/A55 OPPs; both bounds + // are inclusive. + assert!(in_envelope(VDD_FLOOR_UV)); // 675 mV floor, inclusive + assert!(in_envelope(725_000)); + assert!(in_envelope(800_000)); + assert!(in_envelope(VDD_CEIL_UV)); // 1.0 V ceiling, inclusive + assert!(!in_envelope(VDD_FLOOR_UV - 1)); // below floor + assert!(!in_envelope(VDD_CEIL_UV + 1)); // above ceiling + } + + #[test] + fn step_size_is_four_lsb_25mv() { + const STEP_LSB: u8 = (STEP_MAX_UV / VSEL_STEP_UV) as u8; + assert_eq!(STEP_LSB, 4); + assert_eq!((STEP_LSB as u32) * VSEL_STEP_UV, 25_000); + // 800 -> 675 mV is 20 LSB = exactly 5 whole 4-LSB steps. + assert_eq!( + uv_to_vsel(800_000).unwrap() - uv_to_vsel(675_000).unwrap(), + 20 + ); + } +} diff --git a/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs b/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs new file mode 100644 index 0000000000..1985b8cf03 --- /dev/null +++ b/drivers/ax-driver/src/soc/rockchip/pmic_spi.rs @@ -0,0 +1,1115 @@ +// Copyright 2025 The Axvisor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RK3588 A55 (`vdd_cpu_lit`) CPU-rail voltage lever — RK806 PMIC over SPI2. +//! +//! DVFS Phase 2, little cluster. Phase 1a raises the A55 clock via SCMI only; +//! the RK3588 CPU clock is voltage-coupled, so exact frequency + power needs the +//! per-OPP rail voltage that only the PMIC can set. On the OrangePi-5-Plus the +//! A55 rail `vdd_cpu_lit_s0` is provided by an **RK806** PMIC on **SPI2** +//! (`spi@feb20000`, chip-select 0). This module is a minimal polling SPI master +//! plus the RK806 buck-2 regulator access needed to read and (safely, down-only) +//! lower that rail. +//! +//! # Board-confirmed ground truth (sources cited inline) +//! +//! - **SPI2 controller** = `spi@feb20000`, size `0x1000`, compatible +//! `rockchip,rk3066-spi` (the standard Rockchip SPI IP). Register map and the +//! bare-metal polling flow follow mainline `drivers/spi/spi-rockchip.c` and +//! U-Boot `drivers/spi/rk_spi.c` (`rk_spi.h`). +//! - **RK806** = `rockchip,rk806`, CS 0, `spi-max-frequency = 1_000_000`, mode 0. +//! SPI framing from mainline `drivers/mfd/rk8xx-spi.c` +//! (`rk806_spi_bus_read`/`rk806_spi_bus_write`) + constants in +//! `include/linux/mfd/rk808.h`. +//! - **`vdd_cpu_lit_s0` = RK806 `dcdc-reg2` (buck #2)** — from +//! `arch/arm64/boot/dts/rockchip/rk3588-orangepi-5.dtsi`. Its running voltage +//! selector is `RK806_BUCK2_ON_VSEL = 0x1B`, full-byte selector +//! (`vsel_mask = 0xff`), encoded by `rk806_buck_voltage_ranges` in +//! `drivers/regulator/rk808-regulator.c`: +//! `V(uV) = 500000 + sel*6250` for `sel 0..=159` (500 mV..1500 mV, our whole +//! CPU window). 675 mV = sel 28 (`0x1C`), 750 mV = sel 40 (`0x28`), +//! 800 mV = sel 48 (`0x30`). +//! +//! # Safety model +//! +//! Direction is **down only**, toward Linux-proven OPP nominals. [`set_uv`] +//! refuses any target below the [`A55_MIN_UV`] floor (675 mV) or **above the +//! boot voltage read back from the rail** — it can never raise vdd. Every write +//! is read-back verified (raw selector byte compared), and a mismatch aborts and +//! returns `false`, leaving the last confirmed selector in place. +//! [`set_uv_stepped`] lowers in `<=`[`MAX_STEP_UV`] (25 mV) increments with a +//! settle delay so the voltage-coupled clock tracks down gradually. +//! +//! After U-Boot, SPI2 is left with its clocks gated **and** its controller held +//! in soft-reset, so [`init`] first **ungates `PCLK_SPI2` + `CLK_SPI2`**, +//! **de-asserts the SPI2 P+core soft-reset**, and **applies the full SPI2 +//! `pinctrl-0`** (mux + pull + drive + schmitt, via the in-tree rockchip pinctrl +//! driver — a mux-only match is not enough; the pad config is what the RK806 link +//! needs, mirroring the i2c0 fix), then verifies the controller responds (a gated +//! APB reads all-zero, and a core still in reset transacts all-zero — either +//! would otherwise clock in a fabricated `0`); if it does not respond it stays +//! **unbound**. It also logs a one-shot read-only RK806 reachability probe +//! (chip-ID + raw DCDC2 frame). Every SPI poll loop is time-bounded +//! ([`SPI_TIMEOUT_NS`]) and treats an all-zero status register as a dead bus, so +//! a controller that dies after init makes [`get_uv`] return `None` rather than +//! hanging or fabricating a reading. `init`'s only SoC writes are to the SPI2 +//! clock gate, soft-reset, and pin config; **no PMIC write** in `init`. +//! +//! # Integration (owned by the cpufreq probe, not wired here) +//! +//! This module performs **no** PMIC write on its own. Mirroring the A76 I2C +//! sibling, [`init`] maps the controller itself; the probe then reads/steps the +//! rail: +//! +//! ```ignore +//! use crate::soc::rockchip::pmic_spi; +//! +//! pmic_spi::init(); // map + configure SPI2/RK806 +//! let boot_uv = pmic_spi::get_uv(); // read A55 boot vdd first +//! // ... only after the SCMI clock is already at target: +//! pmic_spi::set_uv_stepped(675_000); // down-only to the OPP nominal +//! ``` + +use core::{ + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +// PMIC access is slow: an SPI register read/write busy-waits on the controller +// (~20 ms per transfer). Serialisation must NOT disable preemption across that +// poll, or the caller would stall task-switching on its CPU for the whole +// transaction (the scheduling-starvation risk raised in review). So the slow +// transactions are guarded by a non-preempt-disabling bus-ownership flag +// (`BUS_BUSY` / [`BusGuard`]); the `SpinNoPreempt` `Mutex` below is used ONLY to +// store the controller handle (`init`) or clone it out (`BusGuard::claim`) — a +// microsecond window that never spans a poll. Ownership is single-caller by +// construction (the A55 rail is programmed once at boot; the ring-only governor +// issues no dynamic A55 write), so `BusGuard::claim` bails on the never-expected +// contended case rather than spinning. `SpinNoPreempt` (not `SpinNoIrq`) keeps +// IRQs enabled; the lock is never taken from an interrupt handler. +use ax_kspin::SpinNoPreempt as Mutex; +use log::{info, warn}; +use rdif_pinctrl::PinctrlDevice; + +use crate::mmio::iomap; + +// --------------------------------------------------------------------------- +// SPI2 controller (rockchip,rk3066-spi) — spi@feb20000 +// --------------------------------------------------------------------------- + +/// Physical base of the RK3588 SPI2 controller (`spi@feb20000`). +pub const RK3588_SPI2_BASE: usize = 0xfeb2_0000; +/// MMIO window size of the SPI2 controller. +pub const RK3588_SPI2_SIZE: usize = 0x1000; +/// Fallback SPI2 functional-clock rate for the baud divider, used only if +/// U-Boot left no usable divisor (see [`Rk806Spi::configure`]). +/// +/// The OrangePi-5-Plus DT sets `assigned-clock-rates = <200000000>` for +/// `CLK_SPI2`; 200 MHz is also the RK3588 SPI source ceiling, so assuming it can +/// only *under*-shoot the target SCLK (safe: slower, still <= 1 MHz). +const RK3588_SPI2_FALLBACK_INPUT_HZ: u32 = 200_000_000; + +/// Target serial clock for the RK806 (DT `spi-max-frequency = 1_000_000`). +const SPI_SCLK_HZ: u32 = 1_000_000; +/// RK806 chip-select index on SPI2. +const RK806_CS: u32 = 0; + +// Register offsets (spi-rockchip.c ROCKCHIP_SPI_* / rk_spi.h struct layout). +const SPI_CTRLR0: usize = 0x0000; +const SPI_CTRLR1: usize = 0x0004; +const SPI_ENR: usize = 0x0008; +const SPI_SER: usize = 0x000c; +const SPI_BAUDR: usize = 0x0010; +const SPI_RXFTLR: usize = 0x0018; +const SPI_SR: usize = 0x0024; +const SPI_IMR: usize = 0x002c; +const SPI_ICR: usize = 0x0038; +const SPI_TXDR: usize = 0x0400; +const SPI_RXDR: usize = 0x0800; + +// CTRLR0 fields (rk_spi.h). We now use the EXACT value Linux `spi-rockchip.c` +// programs on this board (read live: 0x2C01), since Linux reads the RK806 with +// it — master, 8-bit frame, mode 0, transmit+receive, EM_BIG. +const CR0_DFS_8BIT: u32 = 0x1; // [1:0] data frame size = 8 bit +const CR0_SSN_DELAY_ONE: u32 = 0x1 << 10; // one sclk CS-to-clk delay +const CR0_EM_BIG: u32 = 0x1 << 11; // endian: big (matches Linux spi-rockchip.c) +const CR0_HALF_WORD_OFF: u32 = 0x1 << 13; // APB 8-bit / SPI 8-bit access +// FRF_SPI (0<<16), TMOD_TR (0<<18), OMOD_MASTER (0<<20), FBM_MSB (0<<12), +// SCPH/SCPOL (mode 0 => 0<<6/0<<7), RXD sample delay 0 all contribute zero. +const SPI_CTRLR0_MODE0_8BIT: u32 = + CR0_DFS_8BIT | CR0_SSN_DELAY_ONE | CR0_EM_BIG | CR0_HALF_WORD_OFF; + +// Status register bits (rk_spi.h). +const SR_BUSY: u32 = 1 << 0; +const SR_TF_FULL: u32 = 1 << 1; +const SR_RF_EMPT: u32 = 1 << 3; + +const BAUDR_MIN: u32 = 2; +const BAUDR_MAX: u32 = 0xfffe; + +// Main RK3588 CRU (write-masked clock/reset registers). U-Boot gates the SPI2 +// clocks after using the RK806 at boot, so `init` must ungate them first — a +// gated APB reads all-zero, which the controller-alive check below rejects. +/// Main RK3588 CRU base. +const RK3588_CRU_BASE: usize = 0xfd7c_0000; +/// CRU window mapped for the ungate (only the first page is needed). +const RK3588_CRU_SIZE: usize = 0x1000; +/// `clkgate_con(14)` (`14*4 + 0x800`): holds the SPI2 gate bits. +const CRU_CLKGATE_CON14: usize = 0x838; +/// Write-masked ungate of `PCLK_SPI2` (bit 8) + `CLK_SPI2` (bit 13): high half +/// is the write-enable mask, low half is the value (0 == clock enabled on a +/// Rockchip CRU gate). == `0x2100_0000`. +const SPI2_CLK_UNGATE: u32 = ((1 << 8) | (1 << 13)) << 16; + +/// Upper bound on any SPI poll loop. Generous vs. the ~40 us a 4-byte 1 MHz +/// frame takes; a dead SPI clock trips this instead of hanging. +const SPI_TIMEOUT_NS: u64 = 20_000_000; // 20 ms + +/// Largest RK806 transaction this module issues: cmd + 2 addr + 1 data. +const RK806_FRAME_LEN: usize = 4; + +// --------------------------------------------------------------------------- +// RK806 SPI framing (rk8xx-spi.c / rk808.h) +// --------------------------------------------------------------------------- + +/// Command byte: write. `RK806_CMD_WRITE = BIT(7)`; CRC disabled (`= 0`); the +/// low nibble carries `value_bytes - 1`, which is 0 for our single-byte access. +const RK806_CMD_WRITE: u8 = 0x80; +/// Command byte: read. `RK806_CMD_READ = 0`; CRC disabled; low nibble 0. +const RK806_CMD_READ: u8 = 0x00; + +/// RK806 buck-2 (`vdd_cpu_lit_s0`) running-voltage selector register. +const RK806_BUCK2_ON_VSEL: u8 = 0x1B; + +/// RK806 chip-name / chip-version registers (rk808.h `RK806_CHIP_NAME`/`_VER`). +/// Both carry nonzero power-on defaults, so a nonzero read proves the SPI path +/// actually reaches the RK806 (vs. reading fabricated zeros off a floating MISO). +const RK806_CHIP_NAME: u8 = 0x5A; +const RK806_CHIP_VER: u8 = 0x5B; + +// --- SPI2 controller soft-reset (RK3588 main CRU) ------------------------- +// Board-observed (boot #2): after ungate the controller is clocked and its APB +// registers stick (`is_alive` passes), yet the serial transaction reads all-zero +// — the classic signature of the controller **core** held in soft-reset: the +// PCLK-domain reset is released (registers work) but the shift-engine reset is +// held, so nothing actually transacts. Same split the i2c0 sibling hit. SPI2's +// resets live in the MAIN CRU: SRST_P_SPI2 (PCLK) = bit 8 and SRST_SPI2 (core) = +// bit 13 of `softrst_con(14)`, from mainline `drivers/clk/rockchip/rst-rk3588.c` +// (`RK3588_CRU_RESET_OFFSET(SRST_P_SPI2,14,8)` / `(SRST_SPI2,14,13)`) — same bit +// layout as the clock gate. We de-assert both (0 == released); the core reset +// (bit 13) is the one that matches the symptom. +/// `softrst_con(14)` (`14*4 + 0xa00`): holds the SPI2 reset bits. +const CRU_SOFTRST_CON14: usize = 0xa38; +/// Write-masked de-assert of SRST_P_SPI2 (bit 8) + SRST_SPI2 (bit 13): high half +/// = write mask, low half = 0 (0 == reset released). Numerically equals the gate +/// ungate value (same bit positions) but targets a different register. +const SPI2_RST_DEASSERT: u32 = ((1 << 8) | (1 << 13)) << 16; +/// Settle after releasing the reset before touching the controller. +const RST_SETTLE_US: u64 = 20; + +// RK3588 PMU1_IOC (@ 0xfd5f0000) GPIO0 A/B pad config, decoded from u-boot +// pinctrl-rk3588.c tables (verified against Linux's live values): +// mux: 0x04 = GPIO0A_H (A4-7), 0x08 = GPIO0B_L (B0-3) [4 bits/pin] +// drive: 0x14 = GPIO0A_H, 0x18 = GPIO0B_L [4 bits/pin] +// pull: 0x20 = GPIO0A, 0x24 = GPIO0B [2 bits/pin] +// smt: 0x30 = GPIO0A, 0x34 = GPIO0B [1 bit/pin] +// SPI2 pins: A5=CLK, A6=MOSI (in 0x04/0x14/0x20/0x30), B1=CS0, B3=MISO (in +// 0x08/0x18/0x24/0x34). The read-only dump below logs this whole region so it +// can be diffed against Linux's known-good reference to find the B3/MISO delta. +const RK3588_IOC_BASE: usize = 0xfd5f_0000; +const RK3588_IOC_SIZE: usize = 0x1000; + +// --------------------------------------------------------------------------- +// RK806 buck voltage encoding (rk806_buck_voltage_ranges, range 1) +// --------------------------------------------------------------------------- + +/// Range-1 base: selector 0 == 500 mV. +const RK806_VSEL_BASE_UV: u32 = 500_000; +/// Range-1 step: 6.25 mV per selector unit. +const RK806_VSEL_STEP_UV: u32 = 6_250; +/// Last selector of range 1 (== 1_493_750 uV; selector 160 is the range-2 base +/// at 1_500_000 uV). Linux's "500mV ~ 1500mV" range comment is approximate. +const RK806_VSEL_R1_MAX_SEL: u8 = 159; +/// Range-2 base (selector 160 == 1_500_000 uV), 25 mV step, up to selector 235. +const RK806_VSEL_R2_BASE_UV: u32 = 1_500_000; +const RK806_VSEL_R2_BASE_SEL: u8 = 160; +const RK806_VSEL_R2_STEP_UV: u32 = 25_000; +const RK806_VSEL_R2_MAX_SEL: u8 = 235; +/// Range-3: selector 236..=255 is a fixed 3_400_000 uV. +const RK806_VSEL_R3_UV: u32 = 3_400_000; + +// --------------------------------------------------------------------------- +// Safety envelope for the A55 rail +// --------------------------------------------------------------------------- + +/// Hard voltage floor for the A55 rail: the lowest Phase-1a OPP nominal +/// (1008 MHz -> 675 mV, standard SKU). Never set below this. Industrial J/M SKU +/// low OPPs sit at 750 mV; a boot read of ~750 is expected there and is fine. +pub const A55_MIN_UV: u32 = 675_000; +/// Maximum single-step magnitude for [`set_uv_stepped`] (25 mV == 4 selectors). +pub const MAX_STEP_UV: u32 = 25_000; +/// Settle delay between stepped-lowering writes (>> the 25 mV ramp time at the +/// DT `regulator-ramp-delay = 12500` uV/us => ~2 us). +const STEP_SETTLE_US: u64 = 150; + +// --------------------------------------------------------------------------- +// Controller handle + global state +// --------------------------------------------------------------------------- + +/// A mapped, initialized SPI2 controller bound to the RK806. +#[derive(Clone)] +struct Rk806Spi { + base: *mut u8, +} + +// The controller handle is only ever stored/cloned under `PMIC` (a microsecond +// `SpinNoPreempt` window) and used while its owner holds `BUS_BUSY`; the MMIO +// mapping is stable for the kernel lifetime. +unsafe impl Send for Rk806Spi {} + +/// Global RK806/SPI2 handle, populated once by [`init`]. The `SpinNoPreempt` +/// mutex is held only to store the handle or clone it out ([`BusGuard::claim`]), +/// never across an SPI poll. +static PMIC: Mutex> = Mutex::new(None); + +/// Non-preempt-disabling bus-ownership flag for the slow RK806 SPI transactions +/// (`rk806_read`/`rk806_write` poll the controller, ~20 ms per transfer). Holding +/// it does not disable preemption, so a transfer can be preempted and never +/// starves the scheduler on that CPU (the review concern). Single-caller by +/// construction (the A55 rail is programmed once at boot by `align_rail_voltages`; +/// the ring-only governor never issues a dynamic A55 voltage write), so +/// [`BusGuard::claim`] takes ownership without blocking and bails on the +/// never-expected contended case instead of spinning. +static BUS_BUSY: AtomicBool = AtomicBool::new(false); + +/// RAII ownership of the RK806 SPI bus for one transaction. Carries a clone of +/// the controller handle so the poll runs with no `SpinNoPreempt` guard held; +/// releases ownership on drop. +struct BusGuard { + dev: Rk806Spi, +} + +impl BusGuard { + /// Take bus ownership without blocking, then clone the handle out under a + /// microsecond `PMIC` lock. `None` if uninitialised or already owned. + fn claim() -> Option { + if BUS_BUSY + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + warn!("pmic_spi: bus already in use; skipping this transaction"); + return None; + } + let handle = PMIC.lock().as_ref().cloned(); + match handle { + Some(dev) => Some(Self { dev }), + None => { + BUS_BUSY.store(false, Ordering::Release); + None + } + } + } +} + +impl Drop for BusGuard { + fn drop(&mut self) { + BUS_BUSY.store(false, Ordering::Release); + } +} + +impl core::ops::Deref for BusGuard { + type Target = Rk806Spi; + fn deref(&self) -> &Rk806Spi { + &self.dev + } +} + +impl Rk806Spi { + #[inline] + fn read(&self, off: usize) -> u32 { + unsafe { self.base.add(off).cast::().read_volatile() } + } + + #[inline] + fn write(&self, off: usize, val: u32) { + unsafe { self.base.add(off).cast::().write_volatile(val) } + } + + /// One-time controller setup: master, mode 0, 8-bit frames, ~1 MHz SCLK. + /// Idempotent; leaves the controller disabled (transfers enable per frame). + /// + /// U-Boot already drives the RK806 over SPI2 at boot, so its BAUDR is a + /// proven-good ~1 MHz divisor — preserve it (like the I2C sibling preserves + /// `CLKDIV`) and only compute one if the controller was left unconfigured. + /// This removes any dependence on knowing the exact `CLK_SPI2` rate. + fn configure(&self) { + // Disable before touching CTRLR0/BAUDR (config is latched while idle). + self.write(SPI_ENR, 0); + self.write(SPI_SER, 0); + + let baud = self.read(SPI_BAUDR) & 0xffff; + if (BAUDR_MIN..=BAUDR_MAX).contains(&baud) && (baud & 1) == 0 { + info!("pmic_spi: SPI2 baud divisor preserved from U-Boot ({baud})"); + } else { + let computed = baud_divider(RK3588_SPI2_FALLBACK_INPUT_HZ, SPI_SCLK_HZ); + self.write(SPI_BAUDR, computed); + info!("pmic_spi: SPI2 baud divisor computed {computed} (U-Boot left {baud:#x})"); + } + + self.write(SPI_CTRLR0, SPI_CTRLR0_MODE0_8BIT); + // Mask + clear any interrupts; this is a poll-only master. + self.write(SPI_IMR, 0); + self.write(SPI_ICR, 0xffff_ffff); + } + + /// Confirm the controller actually responds on its APB. `configure` always + /// writes the fixed `SPI_CTRLR0_MODE0_8BIT`, so those bits reading back + /// proves the bus is clocked and out of reset. A still-gated (or reset-held) + /// controller reads 0 here, which fails the mask check — this is what closes + /// the "gated APB looks like an empty-but-working bus" false-success hole. + fn is_alive(&self) -> bool { + self.read(SPI_CTRLR0) & SPI_CTRLR0_MODE0_8BIT == SPI_CTRLR0_MODE0_8BIT + } + + /// Full-duplex byte exchange of `tx.len()` bytes, CS asserted for the whole + /// transaction. Replicates Linux `spi-rockchip.c`'s exact PIO sequence (the + /// board's known-good driver uses an identical CTRLR0 to ours yet reads the + /// RK806, so the transfer *sequence* is the remaining variable): assert CS, + /// program CTRLR0/CTRLR1/**RXFTLR** while disabled, clear interrupts, enable, + /// **pre-fill the entire TX FIFO**, then drain RX. `rx[3]` holds the value. + /// Returns `false` on timeout, leaving CS deasserted and the controller off. + #[must_use] + fn xfer(&self, tx: &[u8], rx: &mut [u8]) -> bool { + self.xfer_cs(tx, rx, true) + } + + /// As [`Rk806Spi::xfer`], but `native_cs` selects whether the controller's + /// own CS (SER) is asserted. The GPIO-CS diagnostic drives the CS pin + /// manually and calls this with `native_cs = false`. + #[must_use] + fn xfer_cs(&self, tx: &[u8], rx: &mut [u8], native_cs: bool) -> bool { + debug_assert_eq!(tx.len(), rx.len()); + let len = tx.len(); + let n = len as u32; + + // set_cs first (Linux calls set_cs before the transfer). + if native_cs { + self.write(SPI_SER, 1 << RK806_CS); + } + + // Per-transfer config while disabled: CTRLR0, frame count, and the RX + // FIFO threshold (Linux sets RXFTLR = len-1 for small transfers; we + // previously left it unset — a prime suspect). + self.write(SPI_ENR, 0); + self.write(SPI_CTRLR0, SPI_CTRLR0_MODE0_8BIT); + self.write(SPI_CTRLR1, n.saturating_sub(1)); + self.write(SPI_RXFTLR, n.saturating_sub(1)); + self.write(SPI_ICR, 0xffff_ffff); + self.write(SPI_ENR, 1); + + let deadline = now_ns() + SPI_TIMEOUT_NS; + let mut ok = true; + + // Pre-fill the ENTIRE TX FIFO first (Linux fills then waits), rather than + // interleaving TX/RX. The 4-byte frame fits the FIFO with room to spare. + let mut ti = 0usize; + while ti < len { + let sr = self.read(SPI_SR); + if sr == 0 { + warn!("pmic_spi: SPI2 SR reads 0 (controller not clocking); aborting"); + ok = false; + break; + } + if (sr & SR_TF_FULL) == 0 { + self.write(SPI_TXDR, u32::from(tx[ti])); + ti += 1; + } else if now_ns() >= deadline { + warn!("pmic_spi: SPI2 TX fill timed out ({ti}/{len})"); + ok = false; + break; + } + } + + // Then drain the RX FIFO (one byte per clocked frame). + let mut ri = 0usize; + while ok && ri < len { + if (self.read(SPI_SR) & SR_RF_EMPT) == 0 { + rx[ri] = self.read(SPI_RXDR) as u8; + ri += 1; + } else if now_ns() >= deadline { + warn!("pmic_spi: SPI2 RX drain timed out ({ri}/{len})"); + ok = false; + break; + } + } + + // Wait for the shift engine to go idle before dropping CS. + while ok && (self.read(SPI_SR) & SR_BUSY) != 0 { + if now_ns() >= deadline { + warn!("pmic_spi: SPI2 stayed BUSY after transfer"); + ok = false; + break; + } + } + + // Deassert CS0 and disable. + if native_cs { + self.write(SPI_SER, 0); + } + self.write(SPI_ENR, 0); + ok + } + + /// Read one RK806 register (8-bit value, 16-bit little-endian address). + /// Frame: `[READ, addr_lo, addr_hi=0, dummy]`; value is the last RX byte. + fn rk806_read(&self, reg: u8) -> Option { + let tx = [RK806_CMD_READ, reg, 0x00, 0x00]; + let mut rx = [0u8; RK806_FRAME_LEN]; + if self.xfer(&tx, &mut rx) { + Some(rx[RK806_FRAME_LEN - 1]) + } else { + None + } + } + + /// Write one RK806 register. Frame: `[WRITE, addr_lo, addr_hi=0, value]`. + /// RX is ignored. Returns `false` on transfer timeout. + #[must_use] + fn rk806_write(&self, reg: u8, val: u8) -> bool { + let tx = [RK806_CMD_WRITE, reg, 0x00, val]; + let mut rx = [0u8; RK806_FRAME_LEN]; + self.xfer(&tx, &mut rx) + } + + /// Read-only one-shot reachability probe, logged at init. The RK806 + /// chip-name/version registers have nonzero power-on defaults, so a nonzero + /// read proves the SPI path actually reaches the RK806; an all-zero read (and + /// an all-zero DCDC2 frame) means it is not reached (pin-mux / CS). Also dumps + /// the raw DCDC2 read frame for inspection. Performs no PMIC write. + fn log_diagnostics(&self) { + let name = self.rk806_read(RK806_CHIP_NAME); + let ver = self.rk806_read(RK806_CHIP_VER); + let tx = [RK806_CMD_READ, RK806_BUCK2_ON_VSEL, 0x00, 0x00]; + let mut rx = [0u8; RK806_FRAME_LEN]; + let ok = self.xfer(&tx, &mut rx); + info!( + "pmic_spi: RK806 probe: chip_name(0x5A)={name:?} chip_ver(0x5B)={ver:?} \ + dcdc2_raw_rx={rx:02x?} (xfer_ok={ok})" + ); + } + + /// Read-only read-shape diagnostic (boot #8). The RK806 read now returns data + /// but the RX mirrors the TX (e.g. chip-name read `tx=[00,5a,00,00]` came back + /// `rx=[00,5a,00,00]`). Distinguish a MISO-reads-MOSI loopback from a + /// value-position offset: + /// - **distinct**: send a distinctive TX pattern; if the RX echoes it byte for + /// byte, MISO is reading MOSI (pad/routing), not the RK806. + /// - **long6**: a 6-byte frame; if instead the register value lands at a later + /// wire byte (rx\[4]/rx\[5]), it is just a frame-length/index offset. + /// + /// No PMIC write. + fn probe_read_shape(&self) { + let tx1 = [RK806_CMD_READ, RK806_CHIP_NAME, 0xA5, 0x3C]; + let mut rx1 = [0u8; 4]; + let ok1 = self.xfer(&tx1, &mut rx1); + info!("pmic_spi: read-shape distinct: tx={tx1:02x?} rx={rx1:02x?} ok={ok1}"); + + let tx2 = [RK806_CMD_READ, RK806_CHIP_NAME, 0x00, 0x00, 0x00, 0x00]; + let mut rx2 = [0u8; 6]; + let ok2 = self.xfer(&tx2, &mut rx2); + info!("pmic_spi: read-shape long6: tx={tx2:02x?} rx={rx2:02x?} ok={ok2}"); + } +} + +// --------------------------------------------------------------------------- +// Baud + voltage math (pure, unit-tested on the host) +// --------------------------------------------------------------------------- + +/// SPI baud divider: even, in `[2, 0xfffe]`, rounded so the resulting SCLK is +/// `<= target_hz` (mirrors U-Boot `rkspi_set_clk`: `DIV_ROUND_UP` then round up +/// to the next even value). +fn baud_divider(input_hz: u32, target_hz: u32) -> u32 { + let target = target_hz.max(1); + let mut div = input_hz.div_ceil(target); + div = div.clamp(BAUDR_MIN, BAUDR_MAX); + // Round up to the next even number (hardware requires an even divisor); + // rounding up only lowers the clock, which is the safe direction. + div = (div + 1) & 0xfffe; + div.clamp(BAUDR_MIN, BAUDR_MAX) +} + +/// Decode an RK806 buck selector byte to microvolts (all three ranges). +fn vsel_to_uv(sel: u8) -> u32 { + if sel <= RK806_VSEL_R1_MAX_SEL { + RK806_VSEL_BASE_UV + u32::from(sel) * RK806_VSEL_STEP_UV + } else if sel <= RK806_VSEL_R2_MAX_SEL { + RK806_VSEL_R2_BASE_UV + u32::from(sel - RK806_VSEL_R2_BASE_SEL) * RK806_VSEL_R2_STEP_UV + } else { + RK806_VSEL_R3_UV + } +} + +/// Encode microvolts to an RK806 buck selector byte, range 1 only (500 mV.. +/// 1493.75 mV — spanning the entire CPU-rail window). Requires an **exact** +/// 6.25 mV-aligned value (like `pmic_i2c`'s encoder): returns `None` outside +/// range 1 or for any input that does not land exactly on a selector step, so +/// we never program an approximated voltage. Our OPP nominals (675/750/800 mV) +/// are all aligned. +fn uv_to_vsel(uv: u32) -> Option { + let r1_max_uv = RK806_VSEL_BASE_UV + u32::from(RK806_VSEL_R1_MAX_SEL) * RK806_VSEL_STEP_UV; + if uv < RK806_VSEL_BASE_UV || uv > r1_max_uv { + return None; + } + let offset = uv - RK806_VSEL_BASE_UV; + if !offset.is_multiple_of(RK806_VSEL_STEP_UV) { + return None; + } + Some((offset / RK806_VSEL_STEP_UV) as u8) +} + +// --------------------------------------------------------------------------- +// Time helpers +// --------------------------------------------------------------------------- + +#[inline] +fn now_ns() -> u64 { + axklib::time::monotonic_nanos() +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/// Ungate the SPI2 controller clocks (`PCLK_SPI2` + `CLK_SPI2`) in the main CRU. +/// +/// Board-observed: U-Boot gates these after programming the RK806 at boot, so at +/// probe time the SPI2 APB is dead and reads all-zero — which without this looks +/// like a working-but-empty bus and yields a bogus `0` voltage. The Rockchip CRU +/// gate register is write-masked, so writing [`SPI2_CLK_UNGATE`] clears both gate +/// bits (0 == enabled). Returns `false` only if the CRU cannot be mapped. +fn ungate_spi2_clocks() -> bool { + let cru = match iomap(RK3588_CRU_BASE, RK3588_CRU_SIZE) { + Ok(v) => v, + Err(err) => { + warn!("pmic_spi: iomap CRU {RK3588_CRU_BASE:#x} failed: {err:?}"); + return false; + } + }; + // SAFETY: the CRU mapping covers `RK3588_CRU_SIZE` bytes and + // `CRU_CLKGATE_CON14` (0x838) is within the first page. One write-masked + // MMIO store; the gate register tolerates concurrent write-masked updates to + // other bits, so no read-modify-write is needed. + unsafe { + cru.as_ptr() + .add(CRU_CLKGATE_CON14) + .cast::() + .write_volatile(SPI2_CLK_UNGATE); + } + info!( + "pmic_spi: ungated SPI2 PCLK+CLK (CRU {:#x} <= {SPI2_CLK_UNGATE:#010x})", + RK3588_CRU_BASE + CRU_CLKGATE_CON14 + ); + true +} + +/// Release the SPI2 controller from soft-reset in the main CRU so its shift +/// engine can transact. De-assert only (write masked zeros to SRST_P_SPI2 bit 8 + +/// SRST_SPI2 bit 13); we never assert, so nothing else is disturbed, and the +/// BAUDR/CTRLR0 that `configure` (re)writes afterwards take effect on a released +/// core. Idempotent (re-clearing an already-released reset is a no-op). Returns +/// `false` only if the CRU cannot be mapped. +fn deassert_spi2_reset() -> bool { + let cru = match iomap(RK3588_CRU_BASE, RK3588_CRU_SIZE) { + Ok(v) => v, + Err(err) => { + warn!("pmic_spi: iomap CRU {RK3588_CRU_BASE:#x} failed: {err:?}"); + return false; + } + }; + // SAFETY: the CRU mapping covers `RK3588_CRU_SIZE` bytes and + // `CRU_SOFTRST_CON14` (0xa38) is within the first page. One write-masked MMIO + // store clearing only the two SPI2 reset bits. + unsafe { + cru.as_ptr() + .add(CRU_SOFTRST_CON14) + .cast::() + .write_volatile(SPI2_RST_DEASSERT); + } + axklib::time::busy_wait(Duration::from_micros(RST_SETTLE_US)); + info!( + "pmic_spi: de-asserted SPI2 P+core reset (CRU {:#x} <= {SPI2_RST_DEASSERT:#010x})", + RK3588_CRU_BASE + CRU_SOFTRST_CON14 + ); + true +} + +/// Apply the FULL `spi@feb20000` pinctrl-0 (mux **+ pull + drive-strength + +/// schmitt**) via the registered Rockchip pinctrl driver — the same path that +/// fixed i2c0. +/// +/// Board finding (boot #7): our previous mux-only check saw "already func1" yet +/// read AND write to the RK806 were both dead. i2c0 had the identical symptom and +/// was fixed by applying the *full* pin config, not just the mux — the DTS +/// pinconf flags (pull/drive/slew on `spi2m2_pins`/`spi2m2_cs0`) were u-boot's or +/// default, not what the RK806 link needs, so CLK/MOSI/CS never cleanly reached +/// the chip. Rather than hand-decode the pinconf, we apply the node's `pinctrl-0` +/// state through the in-tree pinctrl driver, which already encodes it. Pad-config +/// change only (no PMIC access); best-effort — any failure is logged and the +/// later transaction times out into a safe no-op. +fn set_spi2_pinmux() { + let Some(pinctrl) = rdrive::get_one::() else { + warn!("pmic_spi: PinctrlDevice not registered; cannot apply spi2 pinctrl"); + return; + }; + let mut pinctrl = match pinctrl.lock() { + Ok(guard) => guard, + Err(err) => { + warn!("pmic_spi: failed to lock PinctrlDevice: {err}; cannot apply spi2 pinctrl"); + return; + } + }; + let Some(fdt) = rdrive::with_fdt(Clone::clone) else { + warn!("pmic_spi: live FDT not found; cannot apply spi2 pinctrl"); + return; + }; + // Match the spi2 controller node by reg base (several SPI controllers share + // the rk3066-spi compatible). + let node = fdt + .find_compatible(&["rockchip,rk3066-spi"]) + .into_iter() + .find(|n| { + n.regs() + .into_iter() + .next() + .is_some_and(|r| r.address as usize == RK3588_SPI2_BASE) + }); + let Some(node) = node else { + warn!("pmic_spi: spi2 node @ {RK3588_SPI2_BASE:#x} not found in FDT; cannot apply pinctrl"); + return; + }; + match pinctrl.apply_fdt_default_state(&fdt, node.as_node()) { + Ok(()) => info!( + "pmic_spi: applied spi2 pinctrl-0 (spi2m2 clk/mosi/miso/cs0 mux+pull+drive) via \ + rockchip pinctrl" + ), + Err(err) => warn!("pmic_spi: failed to apply spi2 pinctrl-0: {err:?}"), + } + + // MISO loopback probe (last untested register): force GPIO0_B3 (= SPI2_MISO) to + // INPUT direction. Linux leaves it input (GPIO0 DDR_L bit11 = 0); if StarryOS's + // boot left B3 as a GPIO *output*, the pad would drive instead of sample, so the + // controller reads its own TX shift register — exactly the observed rx==tx + // loopback. The func-1 mux normally overrides GPIO direction, so this is a + // long-shot, but it is the one register we have not yet forced. GPIO0 @ 0xfd8a0000, + // SWPORT_DR_L=0x00, SWPORT_DDR_L=0x08 (write-masked: hi16 = per-bit mask). + if let Ok(gpio0) = iomap(0xfd8a_0000, 0x100) { + let p = gpio0.as_ptr(); + // SAFETY: GPIO0 mapping covers 0x100; DDR_L is at 0x08, within the page. + let ddr = unsafe { p.add(0x08).cast::().read_volatile() }; + info!( + "pmic_spi: GPIO0 DDR_L={:#010x} B3(bit11)={} (0=input) — forcing B3 input", + ddr, + (ddr >> 11) & 1 + ); + // Force B3 -> input: mask bit11 (hi16), value 0 (lo16). + unsafe { p.add(0x08).cast::().write_volatile((1u32 << 11) << 16) }; + } +} + +/// Read-only dump of the applied GPIO0 A/B pad config (boot #9 loopback debug). +/// Logs the whole PMU1_IOC first page for a direct line-by-line diff against +/// Linux's known-good reference, plus the decoded B3/MISO fields (mux 0x08[15:12], +/// drive 0x18[15:12], pull 0x24[7:6], smt 0x34[3]). Linux (works): B3 mux=1, +/// pull=3 (up), smt=0. Whichever field differs on our side is the loopback cause. +/// No writes. +fn dump_spi2_iomux() { + let ioc = match iomap(RK3588_IOC_BASE, RK3588_IOC_SIZE) { + Ok(v) => v.as_ptr(), + Err(err) => { + warn!("pmic_spi: iomap IOC {RK3588_IOC_BASE:#x} failed: {err:?}; skipping IOC dump"); + return; + } + }; + // SAFETY: the IOC mapping covers `RK3588_IOC_SIZE` bytes; every offset read + // below is < 0x44, well within the first page. Read-only. + let rd = |off: usize| unsafe { ioc.add(off).cast::().read_volatile() }; + let (mux_b, drv_b, pull_b, smt_b) = (rd(0x08), rd(0x18), rd(0x24), rd(0x34)); + info!( + "pmic_spi: PMU1_IOC 00={:08x} 04={:08x} 08={:08x} 0c={:08x} 10={:08x} 14={:08x} 18={:08x} \ + 1c={:08x} 20={:08x} 24={:08x} 28={:08x} 2c={:08x} 30={:08x} 34={:08x} 40={:08x}", + rd(0x00), + rd(0x04), + rd(0x08), + rd(0x0c), + rd(0x10), + rd(0x14), + rd(0x18), + rd(0x1c), + rd(0x20), + rd(0x24), + rd(0x28), + rd(0x2c), + rd(0x30), + rd(0x34), + rd(0x40), + ); + info!( + "pmic_spi: B3/MISO decoded: mux={} (want 1) drive={} pull={} (want 3) smt={} (want 0)", + (mux_b >> 12) & 0xf, + (drv_b >> 12) & 0xf, + (pull_b >> 6) & 0x3, + (smt_b >> 3) & 0x1, + ); +} + +/// Map and initialize the RK806-on-SPI2 controller. Idempotent; safe to call +/// from a `PostKernel` probe context (MMU up). +/// +/// Ungates the SPI2 clocks (U-Boot gates them after talking to the RK806 at +/// boot), maps + configures the controller, then verifies it actually responds +/// on its APB. Returns `true` when the controller is bound and ready — including +/// an idempotent re-`init()` of an already-bound controller. Returns `false` on +/// a CRU/SPI2 MMIO mapping failure **or** if the controller does not respond +/// after ungate (still gated, or held in soft-reset — which would need a +/// separate reset de-assert). In every `false` case the controller is left +/// **unbound**, so `get_uv`/`set_uv*` are safe no-ops that leave the boot +/// voltage rather than returning a fabricated `0`. +pub fn init() -> bool { + let mut guard = PMIC.lock(); + if guard.is_some() { + // Already bound: the controller is ready, so report success (not a + // failure) and keep the existing binding. + return true; + } + if !ungate_spi2_clocks() { + return false; + } + let virt = match iomap(RK3588_SPI2_BASE, RK3588_SPI2_SIZE) { + Ok(v) => v, + Err(err) => { + warn!("pmic_spi: iomap {RK3588_SPI2_BASE:#x} failed: {err:?}"); + return false; + } + }; + // SAFETY: `iomap` returned a mapping of `RK3588_SPI2_SIZE` bytes at the SPI2 + // controller base; `base` is only ever used for volatile MMIO within that + // window, serialized through the `PMIC` mutex. + let dev = Rk806Spi { + base: virt.as_ptr(), + }; + // Release the controller core from soft-reset (boot #2: is_alive passed but + // the shift engine read all-zero — core reset held). Must precede configure() + // so BAUDR/CTRLR0 land on a released core. + if !deassert_spi2_reset() { + return false; + } + // Apply the FULL spi2 pinctrl-0 (mux + pull + drive + schmitt) via the + // rockchip pinctrl driver — boot #7 showed a mux-only match isn't enough; the + // pad config is what fixed i2c0. Must precede any transfer. + set_spi2_pinmux(); + // Boot #8: read confirmed MISO loopback. Dump the applied SPI2 mux so we can + // check B3(MISO) is really func1 (vs A6/MOSI which works). + dump_spi2_iomux(); + dev.configure(); + if !dev.is_alive() { + warn!( + "pmic_spi: SPI2 controller not responding after ungate (CTRLR0 read-back failed; APB \ + may still be gated or held in reset); leaving A55 rail untouched" + ); + return false; + } + // Read-only reachability probe (chip-ID + raw DCDC2 frame). + dev.log_diagnostics(); + // Boot #8: the read now returns data but RX mirrors TX — distinguish MISO + // loopback from a value-position offset (read-only). + dev.probe_read_shape(); + info!("pmic_spi: RK806/SPI2 bound at {RK3588_SPI2_BASE:#x}"); + *guard = Some(dev); + true +} + +/// Read the current `vdd_cpu_lit` (A55 rail) voltage in microvolts, or `None` +/// if the controller is not initialized or the SPI read timed out. +pub fn get_uv() -> Option { + let bus = BusGuard::claim()?; + let sel = bus.rk806_read(RK806_BUCK2_ON_VSEL)?; + let uv = vsel_to_uv(sel); + info!("pmic_spi: A55 vdd_cpu_lit = {uv} uV (buck2 vsel {sel:#04x})"); + Some(uv) +} + +/// Set the A55 rail to `target_uv` in a single write, **down only**. +/// +/// Refuses (returns `false`, no write) when: not initialized; the SPI read of +/// the boot voltage fails; `target_uv` is below [`A55_MIN_UV`]; or `target_uv` +/// is above the boot voltage (never raises). On an accepted target it encodes +/// the selector, writes buck-2 `ON_VSEL`, reads it back, and returns `true` only +/// when the read-back selector matches. +// Down-only single-write A55 setter; retained as part of the PMIC-SPI API surface +// for the calibration/voltage-lever path even when the shipped governor build does +// not call it. +#[allow(dead_code)] +pub fn set_uv(target_uv: u32) -> bool { + let Some(bus) = BusGuard::claim() else { + warn!("pmic_spi: set_uv before init or busy; ignored"); + return false; + }; + let dev = &*bus; + let Some(boot_sel) = dev.rk806_read(RK806_BUCK2_ON_VSEL) else { + warn!("pmic_spi: set_uv could not read boot voltage; leaving rail untouched"); + return false; + }; + let boot_uv = vsel_to_uv(boot_sel); + let Some(target_sel) = check_down_only(target_uv, boot_uv) else { + return false; + }; + write_and_verify(dev, target_sel) +} + +/// Lower the A55 rail to `target_uv` in `<=`[`MAX_STEP_UV`] steps, verifying +/// each write and settling between steps, **down only**. +/// +/// Same refusal rules as [`set_uv`]. A `target_uv` equal to the current voltage +/// is an accepted no-op (`true`). Any read-back mismatch aborts mid-descent and +/// returns `false`, leaving the last verified selector in place. +pub fn set_uv_stepped(target_uv: u32) -> bool { + // One claim for the whole stepped transaction (read + step writes). + let Some(bus) = BusGuard::claim() else { + warn!("pmic_spi: set_uv_stepped before init or busy; ignored"); + return false; + }; + let dev = &*bus; + let Some(cur_sel) = dev.rk806_read(RK806_BUCK2_ON_VSEL) else { + warn!("pmic_spi: set_uv_stepped could not read current voltage; leaving rail untouched"); + return false; + }; + let cur_uv = vsel_to_uv(cur_sel); + let Some(target_sel) = check_down_only(target_uv, cur_uv) else { + return false; + }; + + // Higher selector == higher voltage. Down-only means target_sel <= cur_sel. + let step_sel = (MAX_STEP_UV / RK806_VSEL_STEP_UV).max(1) as u8; + let mut sel = cur_sel; + while sel > target_sel { + let next = target_sel.max(sel.saturating_sub(step_sel)); + if !write_and_verify(dev, next) { + return false; + } + axklib::time::busy_wait(Duration::from_micros(STEP_SETTLE_US)); + sel = next; + } + info!( + "pmic_spi: A55 vdd stepped to {} uV (vsel {sel:#04x})", + vsel_to_uv(sel) + ); + true +} + +/// **Diagnostic** force-write of the A55 rail (DCDC2 `ON_VSEL`) — proves whether +/// SPI *writes* physically reach the RK806 while reads are dead. Not part of the +/// normal DVFS API; intended to be called once from a cpufreq test flag. +/// +/// Safety: hard-clamped to `[675_000, 950_000]` uV — the A55 (little cluster) OPP +/// voltage range (675 mV @ 1008 MHz up to 950 mV @ 1800 MHz, all Linux-proven +/// freq/voltage pairs). The governor only ever passes OPP-matched voltages, so any +/// accepted value is a proven-safe rail voltage regardless of the (currently +/// unreadable) present value, and RK3588's voltage-coupled clock tracks the rail in +/// lockstep. Unlike [`set_uv`] it deliberately SKIPS the read-current/down-only +/// guard — the read path is a scope-wall (rx==tx loopback) and the write is what we +/// have. If the SPI path can't reach the RK806, the write is simply a no-op. +/// +/// This is the A55 voltage-set primitive for the ondemand governor (the read-back +/// path is deferred pending the MISO scope fix). Attempts a read-back for the log +/// (`0x00` while reads fail) and returns the write transfer's success. +pub fn force_write_dcdc2(target_uv: u32) -> bool { + if !(675_000..=950_000).contains(&target_uv) { + warn!( + "pmic_spi: force_write_dcdc2 refused target {target_uv} uV (outside A55 OPP range \ + [675000, 950000])" + ); + return false; + } + let Some(bus) = BusGuard::claim() else { + warn!("pmic_spi: force_write_dcdc2 before init or busy; ignored"); + return false; + }; + let dev = &*bus; + let Some(sel) = uv_to_vsel(target_uv) else { + warn!("pmic_spi: force_write_dcdc2 target {target_uv} uV not encodable"); + return false; + }; + let wrote = dev.rk806_write(RK806_BUCK2_ON_VSEL, sel); + let readback = dev.rk806_read(RK806_BUCK2_ON_VSEL); + info!( + "pmic_spi: force_write_dcdc2: wrote vsel={sel:#04x} ({target_uv} uV) xfer_ok={wrote} \ + readback={readback:#04x?}" + ); + wrote +} + +/// Shared down-only clamp: returns the encoded target selector, or `None` +/// (after logging why) when the target must be refused. +fn check_down_only(target_uv: u32, reference_uv: u32) -> Option { + if target_uv < A55_MIN_UV { + warn!("pmic_spi: refusing target {target_uv} uV below floor {A55_MIN_UV} uV"); + return None; + } + if target_uv > reference_uv { + warn!("pmic_spi: refusing to RAISE vdd ({reference_uv} uV -> {target_uv} uV); down-only"); + return None; + } + match uv_to_vsel(target_uv) { + Some(sel) => Some(sel), + None => { + warn!("pmic_spi: target {target_uv} uV not encodable in buck range 1"); + None + } + } +} + +/// Write buck-2 `ON_VSEL` and confirm the read-back selector matches. +fn write_and_verify(dev: &Rk806Spi, sel: u8) -> bool { + if !dev.rk806_write(RK806_BUCK2_ON_VSEL, sel) { + warn!("pmic_spi: buck2 vsel write timed out (sel {sel:#04x})"); + return false; + } + match dev.rk806_read(RK806_BUCK2_ON_VSEL) { + Some(rb) if rb == sel => true, + Some(rb) => { + warn!("pmic_spi: buck2 vsel read-back {rb:#04x} != written {sel:#04x}; aborting"); + false + } + None => { + warn!("pmic_spi: buck2 vsel read-back failed after write; aborting"); + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vsel_decodes_known_opp_nominals() { + assert_eq!(vsel_to_uv(0x1c), 675_000); // sel 28 + assert_eq!(vsel_to_uv(0x28), 750_000); // sel 40 + assert_eq!(vsel_to_uv(0x30), 800_000); // sel 48 + assert_eq!(vsel_to_uv(0), 500_000); + // Selector 159 (range-1 max) is 1_493_750 uV; 1_500_000 is selector 160. + assert_eq!(vsel_to_uv(RK806_VSEL_R1_MAX_SEL), 1_493_750); + } + + #[test] + fn vsel_decodes_upper_ranges() { + assert_eq!(vsel_to_uv(160), 1_500_000); // range 2 base + assert_eq!(vsel_to_uv(161), 1_525_000); + assert_eq!(vsel_to_uv(235), 3_375_000); + assert_eq!(vsel_to_uv(236), 3_400_000); // range 3 fixed + assert_eq!(vsel_to_uv(255), 3_400_000); + } + + #[test] + fn uv_encodes_opp_nominals_exactly() { + assert_eq!(uv_to_vsel(675_000), Some(0x1c)); + assert_eq!(uv_to_vsel(750_000), Some(0x28)); + assert_eq!(uv_to_vsel(800_000), Some(0x30)); + } + + #[test] + fn uv_encode_roundtrips_within_range1() { + for sel in 0..=RK806_VSEL_R1_MAX_SEL { + let uv = vsel_to_uv(sel); + assert_eq!(uv_to_vsel(uv), Some(sel), "sel {sel}"); + } + } + + #[test] + fn uv_encode_rejects_out_of_range1() { + assert_eq!(uv_to_vsel(499_000), None); + assert_eq!(uv_to_vsel(1_600_000), None); + } + + #[test] + fn uv_encode_rejects_unaligned_steps() { + // In-range but not on a 6.25 mV boundary: reject rather than approximate. + assert_eq!(uv_to_vsel(675_000 + 1), None); + assert_eq!(uv_to_vsel(676_000), None); // 176_000 % 6250 != 0 + assert_eq!(uv_to_vsel(500_001), None); + // Boundaries stay accepted. + assert_eq!(uv_to_vsel(500_000), Some(0)); + assert_eq!(uv_to_vsel(506_250), Some(1)); + } + + #[test] + fn down_only_clamp_refuses_below_floor_and_raises() { + // Below the 675 mV floor. + assert_eq!(check_down_only(650_000, 800_000), None); + // Raise attempt (target above reference). + assert_eq!(check_down_only(825_000, 800_000), None); + // Legal down move. + assert_eq!(check_down_only(675_000, 800_000), Some(0x1c)); + // No-op (equal) is allowed. + assert_eq!(check_down_only(800_000, 800_000), Some(0x30)); + } + + #[test] + fn baud_divider_even_and_under_target() { + // 200 MHz functional clock, 1 MHz target -> even divisor, sclk <= 1 MHz. + let div = baud_divider(200_000_000, 1_000_000); + assert_eq!(div % 2, 0); + assert!(200_000_000 / div <= 1_000_000, "div {div}"); + assert_eq!(div, 200); + } + + #[test] + fn baud_divider_clamps_and_stays_even() { + assert_eq!(baud_divider(1_000_000, 1_000_000) % 2, 0); + assert!(baud_divider(u32::MAX, 1) <= BAUDR_MAX); + assert!(baud_divider(1, 1_000_000) >= BAUDR_MIN); + } + + #[test] + fn step_size_is_four_selectors() { + assert_eq!(MAX_STEP_UV / RK806_VSEL_STEP_UV, 4); // 25 mV / 6.25 mV + } + + #[test] + fn spi2_ungate_value_matches_gate_bits() { + // clkgate_con(14): PCLK_SPI2 = bit 8, CLK_SPI2 = bit 13. Write-masked: + // high half selects the bits, low half writes 0 to enable them. + assert_eq!(SPI2_CLK_UNGATE, 0x2100_0000); + assert_eq!(SPI2_CLK_UNGATE & 0xffff, 0); // low half all-0 => enable + assert_eq!(SPI2_CLK_UNGATE >> 16, (1 << 8) | (1 << 13)); + assert_eq!(CRU_CLKGATE_CON14, 14 * 4 + 0x800); // 0x838 + } + + #[test] + fn ctrlr0_mode0_8bit_value() { + // Linux spi-rockchip.c live value on this board: master, mode 0, 8-bit + // DFS, one-cycle CS delay, EM_BIG, APB 8-bit access. + assert_eq!(SPI_CTRLR0_MODE0_8BIT, 0x2c01); + } + + #[test] + fn spi2_reset_deassert_matches_reset_bits() { + // softrst_con(14): SRST_P_SPI2 = bit 8, SRST_SPI2 = bit 13. Write-masked + // de-assert: high half selects the bits, low half writes 0 to release. + assert_eq!(SPI2_RST_DEASSERT, 0x2100_0000); + assert_eq!(SPI2_RST_DEASSERT & 0xffff, 0); // low half all-0 => released + assert_eq!(SPI2_RST_DEASSERT >> 16, (1 << 8) | (1 << 13)); + assert_eq!(CRU_SOFTRST_CON14, 14 * 4 + 0xa00); // 0xa38 + } +} diff --git a/drivers/ax-driver/src/soc/scmi.rs b/drivers/ax-driver/src/soc/scmi.rs index c9586aa7c4..67edf34876 100644 --- a/drivers/ax-driver/src/soc/scmi.rs +++ b/drivers/ax-driver/src/soc/scmi.rs @@ -162,6 +162,44 @@ pub fn set_clock_rate(_phandle: Phandle, clock_id: u32, rate: u64) -> Option<()> } } +/// Query the rates the platform permits for `clock_id` +/// (SCMI `CLOCK_DESCRIBE_RATES`, message 0x4). +/// +/// This is a **read-only** operation: it changes no clock state and is intended +/// as a safety preflight to confirm the firmware actually services a clock +/// before any rate is programmed. Returns `Some(())` when the platform answers +/// the query (the clock exists and its operations are serviced) and `None` when +/// it rejects it. The permitted rates are logged for diagnostics. `_phandle` is +/// ignored (single global agent), mirroring the other helpers here. +pub fn describe_rates(_phandle: Phandle, clock_id: u32) -> Option<()> { + if !SCMI_REGISTERED.load(Ordering::Acquire) { + warn!( + "SCMI describe rates requested before SCMI registration: clock_id={:#x}", + clock_id + ); + return None; + } + let mut guard = SCMI.lock(); + let scmi = guard.as_mut()?; + let mut clock = scmi.protocol_clk_no_init(); + match clock.describe_rates(clock_id, 0) { + Ok(rates) => { + info!( + "SCMI describe rates: clock_id={:#x}, rates={:?}", + clock_id, rates + ); + Some(()) + } + Err(err) => { + warn!( + "SCMI describe rates failed: clock_id={:#x}, {:?}", + clock_id, err + ); + None + } + } +} + struct ScmiDevice; impl DriverGeneric for ScmiDevice { diff --git a/os/StarryOS/configs/board/orangepi-5-plus.toml b/os/StarryOS/configs/board/orangepi-5-plus.toml index 416648573f..31df15cdf5 100644 --- a/os/StarryOS/configs/board/orangepi-5-plus.toml +++ b/os/StarryOS/configs/board/orangepi-5-plus.toml @@ -10,6 +10,7 @@ features = [ "ax-driver/rockchip-ehci", "ax-driver/rockchip-sdhci", "ax-driver/rockchip-dwmmc", + "ax-driver/rk3588-cpufreq", "rknpu", "rk3588-pwm", ] diff --git a/os/StarryOS/kernel/src/entry.rs b/os/StarryOS/kernel/src/entry.rs index ae0363d9fe..5d2823d986 100644 --- a/os/StarryOS/kernel/src/entry.rs +++ b/os/StarryOS/kernel/src/entry.rs @@ -29,6 +29,14 @@ pub fn init(args: &[String], envs: &[String]) { pseudofs::mount_all().expect("Failed to mount pseudofs"); spawn_alarm_task(); + // DVFS: a one-shot OPP-calibration boot runs the sweep and skips the governor; + // otherwise start the ondemand governor. Both run here (early init, before the + // console tty handoff) so their kernel logs reach the serial console. + if ax_driver::cpufreq::calibrate_wanted() { + run_opp_calibration(); + } else { + spawn_cpufreq_governor(); + } pseudofs::usbfs::start_event_pump(); ax_alloc::register_page_reclaim_fn(ax_fs_ng::vfs::page_cache_reclaim); @@ -118,3 +126,68 @@ pub fn init(args: &[String], envs: &[String]) { .flush() .expect("Failed to flush rootfs"); } + +/// Run the one-shot DVFS OPP calibration sweep (gated by the driver's `CALIBRATE` +/// const). Each cluster's (voltage x ring) sweep must execute ON a core of that +/// cluster to read that core's own PMU cycle counter, so we pin a task per cluster +/// (cpu0=A55, cpu4=A76 big0, cpu6=A76 big1) via `set_current_affinity` and run +/// them sequentially (the two A76 rails share one I2C bus). Synchronous: it blocks +/// init briefly so the `CAL` log lines land before the console tty handoff. +fn run_opp_calibration() { + info!("cpufreq: running OPP calibration sweep (governor disabled this boot)"); + for &(cluster_idx, cpu) in &[(0usize, 0usize), (1, 4), (2, 6)] { + let task = ax_task::spawn_raw( + move || { + ax_task::set_current_affinity(ax_task::AxCpuMask::one_shot(cpu)); + ax_driver::cpufreq::calibrate_cluster(cluster_idx, cpu); + }, + String::from("cpufreq-cal"), + ax_task::default_task_stack_size(), + ); + task.join(); + } + info!("cpufreq: OPP calibration sweep complete"); +} + +/// Start the CPU DVFS ondemand governor. +/// +/// The frequency/voltage policy and the SCMI+PMIC apply live in the cpufreq +/// driver (`ax_driver::cpufreq`); this kernel task is only the driver's periodic +/// *loop*. The loop must live here, not in the driver, because ax-driver sits +/// below ax-task/ax-hal in the dependency graph (they pull ax-driver back in via +/// axplat-dyn), so spawning a task inside the driver would be a cyclic dep. Each +/// period we snapshot the per-CPU busy counters the scheduler tick maintains and +/// hand them to `governor_poll`, which decides and applies any OPP change. +/// +/// No-op unless the driver armed the governor (feature on and both CPU-rail PMIC +/// buses up); otherwise every cluster stays on its boot OPP. +fn spawn_cpufreq_governor() { + if !ax_driver::cpufreq::governor_wanted() { + return; + } + info!("Initialize cpufreq ondemand governor..."); + ax_task::spawn_raw( + cpufreq_governor_loop, + String::from("cpufreq-gov"), + ax_task::default_task_stack_size(), + ); +} + +/// Periodic body of the DVFS governor task: sleep, sample every CPU's cumulative +/// busy-tick counter, and let the driver scale each cluster to match load. The +/// slow work (SCMI SMC + PMIC I2C/SPI voltage ramp) happens inside +/// `governor_poll`, which is why this runs in a sleepable task rather than the +/// scheduler tick. +fn cpufreq_governor_loop() { + let period = core::time::Duration::from_millis(ax_driver::cpufreq::governor_period_ms()); + loop { + ax_task::sleep(period); + // RK3588 has 8 CPUs; an offline core's counter never advances, so it + // simply reads as idle (conservative — never over-scales). + let mut busy = [0u64; 8]; + for (cpu, slot) in busy.iter_mut().enumerate() { + *slot = ax_task::cpu_busy_ticks(cpu); + } + ax_driver::cpufreq::governor_poll(&busy); + } +} diff --git a/os/arceos/modules/axtask/src/api.rs b/os/arceos/modules/axtask/src/api.rs index d44f9960e8..399d0f8dee 100644 --- a/os/arceos/modules/axtask/src/api.rs +++ b/os/arceos/modules/axtask/src/api.rs @@ -199,6 +199,19 @@ pub fn next_timer_deadline_nanos() -> Option { crate::timers::next_deadline_nanos() } +/// Scheduler ticks CPU `cpu` has spent running a non-idle task since boot. +/// +/// This is the load metric for an ondemand cpufreq governor: a monotonic per-CPU +/// counter bumped once per timer tick when the CPU is not idle. Sample the delta +/// over a window and divide by the elapsed ticks to get the busy fraction. Returns +/// 0 for an out-of-range `cpu`. Requires the `irq` feature to actually advance +/// (the counter only moves inside the timer tick). +pub fn cpu_busy_ticks(cpu: usize) -> u64 { + crate::run_queue::BUSY_TICKS + .get(cpu) + .map_or(0, |t| t.load(core::sync::atomic::Ordering::Relaxed)) +} + #[cfg(feature = "irq")] #[doc(hidden)] pub fn note_programmed_timer_deadline_nanos(deadline_nanos: u64) { diff --git a/os/arceos/modules/axtask/src/run_queue.rs b/os/arceos/modules/axtask/src/run_queue.rs index b5c839028a..71be34c54d 100644 --- a/os/arceos/modules/axtask/src/run_queue.rs +++ b/os/arceos/modules/axtask/src/run_queue.rs @@ -58,6 +58,15 @@ static mut RUN_QUEUES: [MaybeUninit<&'static mut AxRunQueue>; crate::build_info: #[allow(clippy::declare_interior_mutable_const)] // It's ok because it's used only for initialization `RUN_QUEUES`. const ARRAY_REPEAT_VALUE: MaybeUninit<&'static mut AxRunQueue> = MaybeUninit::uninit(); +/// Per-CPU count of scheduler ticks during which a non-idle task was running, for +/// the ondemand cpufreq governor's load metric. Bumped once per timer tick in +/// [`AxRunQueue::scheduler_timer_tick`] when the current task is not the idle task +/// (that path already runs with IRQ + preempt disabled). Read cross-CPU by the +/// governor via [`crate::cpu_busy_ticks`]; `Relaxed` atomics keep the bump a single +/// instruction on the owning CPU while avoiding a data race on the read. +pub(crate) static BUSY_TICKS: [core::sync::atomic::AtomicU64; crate::build_info::CPU_CAPACITY] = + [const { core::sync::atomic::AtomicU64::new(0) }; crate::build_info::CPU_CAPACITY]; + #[cfg(not(feature = "host-test"))] fn main_task_stack() -> TaskStack { let (stack_ptr, stack_size) = ax_hal::mem::boot_stack_bounds(this_cpu_id()); @@ -640,9 +649,17 @@ impl CurrentRunQueueRef<'_, G> { #[cfg(feature = "irq")] pub fn scheduler_timer_tick(&mut self) { let curr = &self.current_task; - if !curr.is_idle() && self.inner.scheduler.lock().task_tick(curr) { - #[cfg(feature = "preempt")] - curr.set_preempt_pending(true); + if !curr.is_idle() { + // Ondemand-governor load accounting: this CPU ran a real (non-idle) + // task this tick. Already IRQ + preempt off here; a single relaxed + // fetch_add is essentially free. + if let Some(t) = BUSY_TICKS.get(this_cpu_id()) { + t.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + } + if self.inner.scheduler.lock().task_tick(curr) { + #[cfg(feature = "preempt")] + curr.set_preempt_pending(true); + } } } diff --git a/test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml b/test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml index 6e76e2d328..77663747e1 100644 --- a/test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml +++ b/test-suit/starryos/board-orangepi-5-plus/build-aarch64-unknown-none-softfloat.toml @@ -10,6 +10,7 @@ features = [ "ax-driver/rockchip-ehci", "ax-driver/rockchip-sdhci", "ax-driver/rockchip-dwmmc", + "ax-driver/rk3588-cpufreq", "rknpu", "rk3588-pwm", ]