diff --git a/Cargo.lock b/Cargo.lock index 90495e6420..89fa8aaf03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3071,6 +3071,7 @@ dependencies = [ "embedded-hal", "log", "mmio-api", + "rdif-block", "sdmmc-protocol", "volatile 0.6.1", ] @@ -5748,6 +5749,7 @@ dependencies = [ "dma-api", "log", "mmio-api", + "rdif-block", "sdmmc-protocol", "volatile 0.6.1", ] @@ -7166,6 +7168,7 @@ dependencies = [ "embedded-hal", "log", "mmio-api", + "rdif-block", "sdmmc-protocol", ] @@ -7180,6 +7183,7 @@ dependencies = [ "bitflags 2.13.0", "embedded-hal", "log", + "rdif-block", ] [[package]] diff --git a/apps/starry/block-rw-bench/README.md b/apps/starry/block-rw-bench/README.md new file mode 100644 index 0000000000..542468bf60 --- /dev/null +++ b/apps/starry/block-rw-bench/README.md @@ -0,0 +1,11 @@ +# block-rw-bench + +Starry board file-I/O benchmark for SD/MMC RDIF validation. + +Build and install the helper into the board rootfs as `/usr/bin/block-rw-bench` +before running `cargo xtask starry app board -t block-rw-bench -b `. +The board app `init.sh` executes that helper directly. + +The helper writes files under `/root/block-rw-bench/`, calls `sync_all`, reads +the data back, verifies a deterministic pattern, and prints one result line for +each block size plus a final success line. diff --git a/apps/starry/block-rw-bench/block-rw-bench/.gitignore b/apps/starry/block-rw-bench/block-rw-bench/.gitignore new file mode 100644 index 0000000000..ea8c4bf7f3 --- /dev/null +++ b/apps/starry/block-rw-bench/block-rw-bench/.gitignore @@ -0,0 +1 @@ +/target diff --git a/apps/starry/block-rw-bench/block-rw-bench/Cargo.lock b/apps/starry/block-rw-bench/block-rw-bench/Cargo.lock new file mode 100644 index 0000000000..1bf342cee2 --- /dev/null +++ b/apps/starry/block-rw-bench/block-rw-bench/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-rw-bench" +version = "0.1.0" diff --git a/apps/starry/block-rw-bench/block-rw-bench/Cargo.toml b/apps/starry/block-rw-bench/block-rw-bench/Cargo.toml new file mode 100644 index 0000000000..8263527784 --- /dev/null +++ b/apps/starry/block-rw-bench/block-rw-bench/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "block-rw-bench" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" + +[dependencies] + +[workspace] diff --git a/apps/starry/block-rw-bench/block-rw-bench/src/main.rs b/apps/starry/block-rw-bench/block-rw-bench/src/main.rs new file mode 100644 index 0000000000..18033e5f19 --- /dev/null +++ b/apps/starry/block-rw-bench/block-rw-bench/src/main.rs @@ -0,0 +1,136 @@ +use std::{ + env, + fs::{self, File, OpenOptions}, + io::{self, BufReader, Read, Write}, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; + +const BENCH_DIR: &str = "/root/block-rw-bench"; +const TOTAL_BYTES: usize = 64 * 1024 * 1024; +const BLOCK_SIZES: [usize; 3] = [4 * 1024, 64 * 1024, 1024 * 1024]; +const DROP_CACHES_ENV: &str = "BLOCK_RW_BENCH_DROP_CACHES"; + +fn main() { + if let Err(err) = run() { + eprintln!("block-rw-bench: error: {err}"); + std::process::exit(1); + } +} + +fn run() -> io::Result<()> { + let dir = Path::new(BENCH_DIR); + fs::create_dir_all(dir)?; + + for &block_size in &BLOCK_SIZES { + run_case(dir, block_size, TOTAL_BYTES)?; + } + + println!( + "block-rw-bench: done cases={} status=ok", + BLOCK_SIZES.len() + ); + Ok(()) +} + +fn run_case(dir: &Path, block_size: usize, bytes: usize) -> io::Result<()> { + maybe_drop_caches()?; + + let path = case_path(dir, block_size); + let mut pattern = vec![0; block_size]; + let write_start = Instant::now(); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&path)?; + + let mut offset = 0usize; + while offset < bytes { + let chunk_len = (bytes - offset).min(block_size); + fill_pattern(&mut pattern[..chunk_len], block_size, offset); + file.write_all(&pattern[..chunk_len])?; + offset += chunk_len; + } + let write_elapsed = write_start.elapsed(); + + let fsync_start = Instant::now(); + file.sync_all()?; + let fsync_elapsed = fsync_start.elapsed(); + drop(file); + + maybe_drop_caches()?; + + let read_start = Instant::now(); + verify_file(&path, block_size, bytes)?; + let read_elapsed = read_start.elapsed(); + + println!( + "block-rw-bench: case block_size={} bytes={} write_mib_s={:.2} read_mib_s={:.2} fsync_ms={} verify=ok", + block_size, + bytes, + throughput_mib_s(bytes, write_elapsed), + throughput_mib_s(bytes, read_elapsed), + duration_ms(fsync_elapsed) + ); + + fs::remove_file(path)?; + Ok(()) +} + +fn verify_file(path: &Path, block_size: usize, bytes: usize) -> io::Result<()> { + let mut reader = BufReader::new(File::open(path)?); + let mut actual = vec![0; block_size]; + let mut expected = vec![0; block_size]; + let mut offset = 0usize; + + while offset < bytes { + let chunk_len = (bytes - offset).min(block_size); + reader.read_exact(&mut actual[..chunk_len])?; + fill_pattern(&mut expected[..chunk_len], block_size, offset); + if actual[..chunk_len] != expected[..chunk_len] { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "verify mismatch block_size={} offset={} expected={:02x} actual={:02x}", + block_size, offset, expected[0], actual[0] + ), + )); + } + offset += chunk_len; + } + + Ok(()) +} + +fn fill_pattern(buf: &mut [u8], block_size: usize, base_offset: usize) { + let seed = block_size as u64 ^ 0x5d51_d1f5_a5a5_1234; + for (index, byte) in buf.iter_mut().enumerate() { + let pos = (base_offset + index) as u64; + *byte = pos + .wrapping_mul(1103515245) + .wrapping_add(seed) + .rotate_left((pos & 7) as u32) as u8; + } +} + +fn throughput_mib_s(bytes: usize, elapsed: Duration) -> f64 { + let seconds = elapsed.as_secs_f64().max(0.000_001); + bytes as f64 / (1024.0 * 1024.0) / seconds +} + +fn duration_ms(elapsed: Duration) -> u128 { + elapsed.as_millis() +} + +fn case_path(dir: &Path, block_size: usize) -> PathBuf { + dir.join(format!("case-{}.bin", block_size)) +} + +fn maybe_drop_caches() -> io::Result<()> { + if env::var_os(DROP_CACHES_ENV).is_none() { + return Ok(()); + } + + fs::write("/proc/sys/vm/drop_caches", b"3\n") +} diff --git a/apps/starry/block-rw-bench/board-orangepi-5-plus.toml b/apps/starry/block-rw-bench/board-orangepi-5-plus.toml new file mode 100644 index 0000000000..7554d43b03 --- /dev/null +++ b/apps/starry/block-rw-bench/board-orangepi-5-plus.toml @@ -0,0 +1,13 @@ +board_type = "OrangePi-5-Plus" +shell_prefix = "root@starry:/root #" +success_regex = [ + '(?m)^block-rw-bench: done cases=3 status=ok$', +] +fail_regex = [ + '(?i)\bpanic(?:ked)?\b', + '(?m)^block-rw-bench: error: .*$', + '(?m)^block-rw-bench: .*verify=(?:fail|mismatch).*$', + '(?i)(block-rw-bench|sh): .*not found', + '(?i)\bI/O error\b', +] +timeout = 600 diff --git a/apps/starry/block-rw-bench/board-phytiumpi.toml b/apps/starry/block-rw-bench/board-phytiumpi.toml new file mode 100644 index 0000000000..52c2b3ae08 --- /dev/null +++ b/apps/starry/block-rw-bench/board-phytiumpi.toml @@ -0,0 +1,13 @@ +board_type = "PhytiumPi" +shell_prefix = "root@starry:/root #" +success_regex = [ + '(?m)^block-rw-bench: done cases=3 status=ok$', +] +fail_regex = [ + '(?i)\bpanic(?:ked)?\b', + '(?m)^block-rw-bench: error: .*$', + '(?m)^block-rw-bench: .*verify=(?:fail|mismatch).*$', + '(?i)(block-rw-bench|sh): .*not found', + '(?i)\bI/O error\b', +] +timeout = 600 diff --git a/apps/starry/block-rw-bench/board-roc-rk3568-pc.toml b/apps/starry/block-rw-bench/board-roc-rk3568-pc.toml new file mode 100644 index 0000000000..0043a5fa16 --- /dev/null +++ b/apps/starry/block-rw-bench/board-roc-rk3568-pc.toml @@ -0,0 +1,13 @@ +board_type = "ROC-RK3568-PC" +shell_prefix = "root@starry:/root #" +success_regex = [ + '(?m)^block-rw-bench: done cases=3 status=ok$', +] +fail_regex = [ + '(?i)\bpanic(?:ked)?\b', + '(?m)^block-rw-bench: error: .*$', + '(?m)^block-rw-bench: .*verify=(?:fail|mismatch).*$', + '(?i)(block-rw-bench|sh): .*not found', + '(?i)\bI/O error\b', +] +timeout = 1200 diff --git a/apps/starry/block-rw-bench/build-aarch64-unknown-none-softfloat.toml b/apps/starry/block-rw-bench/build-aarch64-unknown-none-softfloat.toml new file mode 100644 index 0000000000..53e7e53daa --- /dev/null +++ b/apps/starry/block-rw-bench/build-aarch64-unknown-none-softfloat.toml @@ -0,0 +1,9 @@ +target = "aarch64-unknown-none-softfloat" +features = [ + "ax-driver/rockchip-soc", + "ax-driver/rockchip-sdhci", + "ax-driver/rockchip-dwmmc", + "ax-driver/phytium-mci", +] +log = "Info" +max_cpu_num = 1 diff --git a/apps/starry/block-rw-bench/init.sh b/apps/starry/block-rw-bench/init.sh new file mode 100644 index 0000000000..73f919c280 --- /dev/null +++ b/apps/starry/block-rw-bench/init.sh @@ -0,0 +1 @@ +rm -rf /root/block-rw-bench && mkdir -p /root/block-rw-bench && /usr/bin/block-rw-bench && sync diff --git a/drivers/ax-driver/Cargo.toml b/drivers/ax-driver/Cargo.toml index 0a42983940..642f83e189 100644 --- a/drivers/ax-driver/Cargo.toml +++ b/drivers/ax-driver/Cargo.toml @@ -66,7 +66,7 @@ rockchip-sdhci = [ "dep:rdif-clk", "dep:sdhci-host", "dep:sdmmc-protocol", - "sdmmc-protocol/sdio", + "sdmmc-protocol/rdif", ] phytium-mci = [ "block", @@ -74,14 +74,14 @@ phytium-mci = [ "dep:ax-kspin", "dep:phytium-mci-host", "dep:sdmmc-protocol", - "sdmmc-protocol/sdio", + "sdmmc-protocol/rdif", ] k230-sdhci = [ "block", "plat-dyn", "dep:sdhci-host", "dep:sdmmc-protocol", - "sdmmc-protocol/sdio", + "sdmmc-protocol/rdif", ] rockchip-dwmmc = [ "block", @@ -92,7 +92,7 @@ rockchip-dwmmc = [ "dep:dwmmc-host", "dep:rdif-clk", "dep:sdmmc-protocol", - "sdmmc-protocol/sdio", + "sdmmc-protocol/rdif", ] rk3588-pcie = [ "pci", diff --git a/drivers/ax-driver/src/block/k230_sdhci.rs b/drivers/ax-driver/src/block/k230_sdhci.rs index 3d807825a1..5098d08988 100644 --- a/drivers/ax-driver/src/block/k230_sdhci.rs +++ b/drivers/ax-driver/src/block/k230_sdhci.rs @@ -20,20 +20,14 @@ use rdrive::{ probe::OnProbeError, register::{FdtInfo, ProbeFdt}, }; -use sdhci_host::Sdhci; +use sdhci_host::{Sdhci, rdif as sdhci_rdif}; use sdmmc_protocol::{ Error, OperationPoll, error::Phase, sdio::{CardInfo, CardInitPreference, SdioInitScratch, SdioSdmmc}, }; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap}; const SDHCI_POWER_330: u8 = 0x0e; @@ -78,7 +72,8 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { .map_err(|e| init_error(base_reg.address, mmio_size, e))?; host.set_power(SDHCI_POWER_330); host.enable_interrupts(); - host.set_dma(axklib::dma::device_with_mask(u32::MAX as u64)); + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); info!("k230-sdhci: initialize card"); let mut card = SdioSdmmc::new(host); @@ -96,10 +91,14 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(card); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::dma("k230-sdhci", card_info.capacity_blocks.unwrap_or(0), false), + let dev = sdhci_rdif::device( + card, + sdhci_rdif::dma_config( + "k230-sdhci", + card_info.capacity_blocks.unwrap_or(0), + false, + dma, + ), ); let irq = probe.register_block(dev)?; info!("k230-sdhci block device registered irq={:?}", irq); diff --git a/drivers/ax-driver/src/block/mod.rs b/drivers/ax-driver/src/block/mod.rs index 691f354c46..0e10d0bcf2 100644 --- a/drivers/ax-driver/src/block/mod.rs +++ b/drivers/ax-driver/src/block/mod.rs @@ -1,12 +1,5 @@ mod binding; -#[cfg(any( - feature = "k230-sdhci", - feature = "phytium-mci", - feature = "rockchip-dwmmc", - feature = "rockchip-sdhci" -))] -pub(crate) mod sdmmc; #[allow(unused)] mod shared; diff --git a/drivers/ax-driver/src/block/phytium_mci.rs b/drivers/ax-driver/src/block/phytium_mci.rs index 26c9e93f7e..9029968696 100644 --- a/drivers/ax-driver/src/block/phytium_mci.rs +++ b/drivers/ax-driver/src/block/phytium_mci.rs @@ -2,7 +2,7 @@ use alloc::format; use core::time::Duration; use log::{info, warn}; -use phytium_mci_host::PhytiumMci; +use phytium_mci_host::{PhytiumMci, rdif as phytium_rdif}; use rdrive::{ probe::OnProbeError, register::{FdtInfo, ProbeFdt}, @@ -13,13 +13,7 @@ use sdmmc_protocol::{ sdio::{CardInfo, CardInitPreference, SdioInitScratch, SdioSdmmc}, }; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap}; type PhytiumSdMmc = SdioSdmmc; @@ -81,10 +75,14 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(card); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::fifo("phytium-mci", card_info.capacity_blocks.unwrap_or(0), false), + let dev = phytium_rdif::device( + card, + phytium_rdif::dma_config( + "phytium-mci", + card_info.capacity_blocks.unwrap_or(0), + false, + axklib::dma::device_with_mask(u32::MAX as u64), + ), ); let irq = probe.register_block(dev)?; info!("phytium-mci block device registered irq={:?}", irq); diff --git a/drivers/ax-driver/src/block/rockchip/sdhci_rk3568.rs b/drivers/ax-driver/src/block/rockchip/sdhci_rk3568.rs index 71fdbefa26..2137cf9881 100644 --- a/drivers/ax-driver/src/block/rockchip/sdhci_rk3568.rs +++ b/drivers/ax-driver/src/block/rockchip/sdhci_rk3568.rs @@ -22,7 +22,7 @@ use rdrive::{ probe::OnProbeError, register::{FdtInfo, ProbeFdt}, }; -use sdhci_host::{HostClock, Sdhci}; +use sdhci_host::{HostClock, Sdhci, rdif as sdhci_rdif}; use sdmmc_protocol::{ Error, OperationPoll, error::{ErrorContext, Phase}, @@ -30,13 +30,7 @@ use sdmmc_protocol::{ }; use spin::Once; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap}; const SDHCI_POWER_330: u8 = 0x0e; @@ -146,7 +140,8 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { init_dwcmshc_after_reset(mmio_base); host.set_power(SDHCI_POWER_330); host.enable_interrupts(); - host.set_dma(axklib::dma::device_with_mask(u32::MAX as u64)); + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); info!("rockchip-rk3568-sdhci: initialize card"); let mut card = SdioSdmmc::new(host); @@ -164,13 +159,13 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(card); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::fifo( + let dev = sdhci_rdif::device( + card, + sdhci_rdif::dma_config( "rockchip-rk3568-sdhci", card_info.capacity_blocks.unwrap_or(0), true, + dma, ), ); let irq = probe.register_block(dev)?; @@ -383,19 +378,30 @@ mod tests { use super::*; #[test] - fn rk3568_block_io_uses_fifo_config() { - let config = SdmmcBlockConfig::fifo("rockchip-rk3568-sdhci", 8, true); + fn rk3568_block_io_uses_dma_config() { + let config = sdhci_rdif::dma_config( + "rockchip-rk3568-sdhci", + 8, + true, + axklib::dma::device_with_mask(u32::MAX as u64), + ); - assert!(!config.use_dma); + assert!(config.uses_dma()); + assert!(config.irq_driven); } #[test] - fn rk3568_fifo_queue_limits_single_block_requests() { - let config = SdmmcBlockConfig::fifo("rockchip-rk3568-sdhci", 8, true); - let limits = crate::block::sdmmc::queue_limits(&config, u32::MAX as u64); + fn rk3568_dma_queue_limits_multi_block_requests() { + let config = sdhci_rdif::dma_config( + "rockchip-rk3568-sdhci", + 8, + true, + axklib::dma::device_with_mask(u32::MAX as u64), + ); + let limits = sdmmc_protocol::rdif::queue_limits(&config, u32::MAX as u64); - assert_eq!(limits.max_blocks_per_request, 1); - assert_eq!(limits.max_segment_size, crate::block::sdmmc::BLOCK_SIZE); + assert!(limits.max_blocks_per_request > 1); + assert!(limits.max_segment_size > sdmmc_protocol::rdif::BLOCK_SIZE); assert_eq!(limits.max_segments, 1); } } diff --git a/drivers/ax-driver/src/block/rockchip_mmc.rs b/drivers/ax-driver/src/block/rockchip_mmc.rs index dedb26e8e4..96a78212cc 100644 --- a/drivers/ax-driver/src/block/rockchip_mmc.rs +++ b/drivers/ax-driver/src/block/rockchip_mmc.rs @@ -22,7 +22,7 @@ use rdrive::{ probe::OnProbeError, register::{FdtInfo, ProbeFdt}, }; -use sdhci_host::{HostClock, Sdhci}; +use sdhci_host::{HostClock, Sdhci, rdif as sdhci_rdif}; use sdmmc_protocol::{ Error, OperationPoll, error::{ErrorContext, Phase}, @@ -30,13 +30,7 @@ use sdmmc_protocol::{ }; use spin::Once; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap}; const SDHCI_POWER_330: u8 = 0x0e; static SDHCI_CLOCK: RockchipSdhciClock = RockchipSdhciClock; @@ -98,7 +92,8 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { .map_err(|e| init_error(base_reg.address, mmio_size, e))?; host.set_power(SDHCI_POWER_330); host.enable_interrupts(); - host.set_dma(axklib::dma::device_with_mask(u32::MAX as u64)); + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); info!("rockchip-sdhci: initialize card"); let mut card = SdioSdmmc::new(host); @@ -116,13 +111,13 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(card); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::dma( + let dev = sdhci_rdif::device( + card, + sdhci_rdif::dma_config( "rockchip-sdhci", card_info.capacity_blocks.unwrap_or(0), true, + dma, ), ); let irq = probe.register_block(dev)?; diff --git a/drivers/ax-driver/src/block/rockchip_sd.rs b/drivers/ax-driver/src/block/rockchip_sd.rs index 957f43dd7f..4ee983357a 100644 --- a/drivers/ax-driver/src/block/rockchip_sd.rs +++ b/drivers/ax-driver/src/block/rockchip_sd.rs @@ -15,7 +15,7 @@ use alloc::format; use core::time::Duration; -use dwmmc_host::DwMmc; +use dwmmc_host::{DwMmc, rdif as dwmmc_rdif}; use log::{info, warn}; use rdif_clk::ClockId; use rdrive::{ @@ -28,14 +28,7 @@ use sdmmc_protocol::{ sdio::{CardInfo, SdioInitScratch, SdioSdmmc}, }; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, - soc::scmi, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap, soc::scmi}; const DWMMC_STABLE_REFERENCE_CLOCK: u32 = 50_000_000; const ENABLE_SD_SPEED_SELECTION: bool = true; @@ -107,7 +100,8 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { info!("rockchip-dwmmc: reset controller"); host.reset_and_init() .map_err(|e| init_error(base_reg.address, mmio_size, e))?; - host.set_dma(axklib::dma::device_with_mask(u32::MAX as u64)); + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); info!("rockchip-dwmmc: initialize card"); let mut sd = SdioSdmmc::new(host); @@ -134,10 +128,14 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { tune_rk3588_sdmmc_sample_phase(&mut sd, reference_clock); } - let raw = SharedDriver::new(sd); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::dma("rockchip-sd", card_info.capacity_blocks.unwrap_or(0), true), + let dev = dwmmc_rdif::device( + sd, + dwmmc_rdif::dma_config( + "rockchip-sd", + card_info.capacity_blocks.unwrap_or(0), + true, + dma, + ), ); let irq = probe.register_block(dev)?; info!("rockchip-sd block device registered irq={:?}", irq); diff --git a/drivers/ax-driver/src/block/rockchip_sd/phase.rs b/drivers/ax-driver/src/block/rockchip_sd/phase.rs index 7e2e37052a..d13f1255fe 100644 --- a/drivers/ax-driver/src/block/rockchip_sd/phase.rs +++ b/drivers/ax-driver/src/block/rockchip_sd/phase.rs @@ -2,14 +2,14 @@ use core::ptr::NonNull; use log::{info, warn}; use rdrive::{probe::OnProbeError, register::FdtInfo}; -use sdmmc_protocol::{DataCommandPoll, Error}; +use sdmmc_protocol::{DataCommandPoll, Error, rdif::BLOCK_SIZE}; use super::{ RK3588_CRU_BASE, RK3588_CRU_SIZE, RK3588_SDMMC_CON0, RK3588_SDMMC_CON1, RK3588_SDMMC_DRV_PHASE_DEG, RK3588_SDMMC_PHASE_SHIFT, RK3588_SDMMC_SAMPLE_PHASE_CANDIDATES, RK3588_SDMMC_SAMPLE_PHASE_DEG, RockchipDwMmc, }; -use crate::{block::sdmmc::BLOCK_SIZE, mmio::iomap}; +use crate::mmio::iomap; pub(super) fn init_rk3588_sdmmc_phase( info: &FdtInfo<'_>, diff --git a/drivers/ax-driver/src/block/sdmmc.rs b/drivers/ax-driver/src/block/sdmmc.rs deleted file mode 100644 index ef7a972f7a..0000000000 --- a/drivers/ax-driver/src/block/sdmmc.rs +++ /dev/null @@ -1,1079 +0,0 @@ -use alloc::{boxed::Box, vec, vec::Vec}; -use core::{ - marker::PhantomData, - num::NonZeroUsize, - ptr::NonNull, - sync::atomic::{AtomicBool, Ordering}, -}; - -use dma_api::DeviceDma; -use log::warn; -use rdrive::DriverGeneric; -use sdmmc_protocol::{ - BlockPoll, BlockRequestId, Error, - sdio::{SdioHost, SdioIrqHandle, SdioIrqHost, SdioSdmmc, block_queue_ready_from_host_event}, -}; - -use crate::block::SharedDriver; - -pub(crate) const BLOCK_SIZE: usize = 512; - -#[derive(Clone, Copy)] -pub(crate) struct SdmmcBlockConfig { - pub name: &'static str, - pub capacity_blocks: u64, - pub dma_mask: u64, - pub max_blocks_per_request: u32, - pub max_segment_size: usize, - pub irq_driven: bool, - pub use_dma: bool, -} - -impl SdmmcBlockConfig { - #[cfg(any( - feature = "k230-sdhci", - feature = "rockchip-dwmmc", - feature = "rockchip-sdhci", - test - ))] - pub(crate) fn dma(name: &'static str, capacity_blocks: u64, irq_driven: bool) -> Self { - Self { - name, - capacity_blocks, - dma_mask: u32::MAX as u64, - max_blocks_per_request: u16::MAX as u32 + 1, - max_segment_size: usize::MAX, - irq_driven, - use_dma: true, - } - } - - #[cfg(any(feature = "rockchip-sdhci", feature = "phytium-mci", test))] - pub(crate) fn fifo(name: &'static str, capacity_blocks: u64, irq_driven: bool) -> Self { - Self { - name, - capacity_blocks, - dma_mask: u32::MAX as u64, - max_blocks_per_request: 1, - max_segment_size: BLOCK_SIZE, - irq_driven, - use_dma: false, - } - } -} - -pub(crate) trait SdmmcBlockHost: SdioIrqHost + Send + Sync + 'static { - type Request: Send + 'static; - type Slot: Default + Send + 'static; - - fn submit_read_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result; - - fn submit_write_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result; - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - slot: &mut Self::Slot, - ) -> Result; - - fn request_id(request: &Self::Request) -> BlockRequestId; -} - -pub(crate) struct SdmmcBlockDevice -where - H: SdmmcBlockHost, -{ - raw: Option>>, - irq_handle: ::IrqHandle, - config: SdmmcBlockConfig, - irq_enabled: AtomicBool, - queue_created: bool, - irq_handler_taken: bool, -} - -impl SdmmcBlockDevice -where - H: SdmmcBlockHost, -{ - pub(crate) fn new(raw: SharedDriver>, config: SdmmcBlockConfig) -> Self { - let irq_handle = raw.with_mut(|raw| raw.host().irq_handle()); - Self { - raw: Some(raw), - irq_handle, - config, - irq_enabled: AtomicBool::new(false), - queue_created: false, - irq_handler_taken: false, - } - } - - fn queue_limits_with_mask(&self, dma_mask: u64) -> rdif_block::QueueLimits { - queue_limits(&self.config, dma_mask) - } -} - -impl DriverGeneric for SdmmcBlockDevice -where - H: SdmmcBlockHost, -{ - fn name(&self) -> &str { - self.config.name - } -} - -impl rdif_block::Interface for SdmmcBlockDevice -where - H: SdmmcBlockHost, -{ - fn device_info(&self) -> rdif_block::DeviceInfo { - device_info(&self.config) - } - - fn queue_limits(&self) -> rdif_block::QueueLimits { - self.queue_limits_with_mask(self.config.dma_mask) - } - - fn create_queue(&mut self) -> Option> { - if self.queue_created { - return None; - } - self.raw.clone().as_ref().map(|dev| { - self.queue_created = true; - Box::new(SdmmcBlockQueue::::new( - dev.clone(), - self.config.name, - self.config.capacity_blocks, - self.config, - 0, - )) as _ - }) - } - - fn enable_irq(&self) { - if !self.config.irq_driven { - self.irq_enabled.store(false, Ordering::Release); - return; - } - if let Some(raw) = &self.raw { - let mut enabled = false; - raw.with_mut(|raw| { - if let Err(err) = SdioHost::enable_completion_irq(raw.host_mut()) { - warn!( - "{}: enable completion IRQ failed: {:?}", - self.config.name, err - ); - return; - } - enabled = raw.host().completion_irq_enabled(); - }); - self.irq_enabled.store(enabled, Ordering::Release); - } - } - - fn disable_irq(&self) { - if let Some(raw) = &self.raw { - raw.with_mut(|raw| { - if let Err(err) = SdioHost::disable_completion_irq(raw.host_mut()) { - warn!( - "{}: disable completion IRQ failed: {:?}", - self.config.name, err - ); - } - }); - } - self.irq_enabled.store(false, Ordering::Release); - } - - fn is_irq_enabled(&self) -> bool { - self.irq_enabled.load(Ordering::Acquire) - } - - fn irq_sources(&self) -> rdif_block::IrqSourceList { - if !self.config.irq_driven { - return Vec::new(); - } - vec![rdif_block::IrqSourceInfo::legacy( - rdif_block::IdList::from_bits(1), - )] - } - - fn take_irq_handler(&mut self, source_id: usize) -> Option> { - if !self.config.irq_driven || source_id != 0 { - return None; - } - if self.irq_handler_taken { - return None; - } - self.irq_handler_taken = true; - Some(Box::new(SdmmcBlockIrqHandler:: { - handle: self.irq_handle.clone(), - _marker: PhantomData, - })) - } -} - -struct SdmmcBlockQueue -where - H: SdmmcBlockHost, -{ - raw: SharedDriver>, - name: &'static str, - capacity_blocks: u64, - config: SdmmcBlockConfig, - id: usize, - dma: DeviceDma, - slot: H::Slot, - pending: Option, - completed: Vec, -} - -impl SdmmcBlockQueue -where - H: SdmmcBlockHost, -{ - fn new( - raw: SharedDriver>, - name: &'static str, - capacity_blocks: u64, - config: SdmmcBlockConfig, - id: usize, - ) -> Self { - let dma_mask = config.dma_mask; - Self { - raw, - name, - capacity_blocks, - config, - id, - dma: axklib::dma::device_with_mask(dma_mask), - slot: H::Slot::default(), - pending: None, - completed: Vec::new(), - } - } - - fn queue_info(&self) -> rdif_block::QueueInfo { - rdif_block::IQueue::info(self) - } - - fn submit_request_inner( - &mut self, - request: rdif_block::Request<'_>, - ) -> Result { - rdif_block::validate_request(self.queue_info(), &request)?; - self.reap_pending_request()?; - let raw = self.raw.clone(); - raw.with_mut(|raw| { - let start_block = block_addr_for_card(request.lba, raw.is_high_capacity())?; - let buffer = request - .segments - .first() - .copied() - .ok_or(rdif_block::BlkError::InvalidRequest)?; - if !buffer.len().is_multiple_of(BLOCK_SIZE) { - return Err(rdif_block::BlkError::Other("buffer is not block aligned")); - } - let ptr = NonNull::new(buffer.virt) - .ok_or(rdif_block::BlkError::Other("buffer pointer is null"))?; - let size = NonZeroUsize::new(buffer.len()) - .ok_or(rdif_block::BlkError::Other("buffer is empty"))?; - let id = match request.op { - rdif_block::RequestOp::Read => H::submit_read_request( - raw.host_mut(), - start_block, - ptr, - size, - transfer_dma(self.config.use_dma, &self.dma), - &mut self.slot, - &mut self.pending, - )?, - rdif_block::RequestOp::Write => H::submit_write_request( - raw.host_mut(), - start_block, - ptr, - size, - transfer_dma(self.config.use_dma, &self.dma), - &mut self.slot, - &mut self.pending, - )?, - rdif_block::RequestOp::Flush - | rdif_block::RequestOp::Discard - | rdif_block::RequestOp::WriteZeroes => { - return Err(rdif_block::BlkError::NotSupported); - } - }; - Ok(rdif_block::RequestId::new(usize::from(id))) - }) - } - - fn poll_request_inner( - &mut self, - request: rdif_block::RequestId, - ) -> Result { - if let Some(index) = self.completed.iter().position(|id| *id == request) { - self.completed.swap_remove(index); - return Ok(rdif_block::RequestStatus::Complete); - } - self.poll_active_request(request) - } - - fn poll_active_request( - &mut self, - request: rdif_block::RequestId, - ) -> Result { - let raw = self.raw.clone(); - match raw.with_mut(|raw| { - H::poll_block_request( - raw.host_mut(), - &mut self.pending, - BlockRequestId::new(usize::from(request)), - &mut self.slot, - ) - }) { - Ok(BlockPoll::Complete) => Ok(rdif_block::RequestStatus::Complete), - Ok(BlockPoll::Pending) => Ok(rdif_block::RequestStatus::Pending), - Ok(_) => Err(rdif_block::BlkError::Other( - "SD/MMC returned an unknown poll state", - )), - Err(err) => Err(map_dev_err_to_blk_err(err)), - } - } - - fn pending_id(&self) -> Option { - self.pending.as_ref().map(H::request_id) - } - - fn reap_pending_request(&mut self) -> Result { - let Some(active) = self.pending_id() else { - return Ok(rdif_block::RequestStatus::Complete); - }; - let id = rdif_block::RequestId::new(usize::from(active)); - match self.poll_active_request(id) { - Ok(rdif_block::RequestStatus::Complete) => { - self.completed.push(id); - Ok(rdif_block::RequestStatus::Complete) - } - Ok(rdif_block::RequestStatus::Pending) => Err(rdif_block::BlkError::Retry), - Err(err) => Err(err), - } - } -} - -// SAFETY: `SdmmcBlockQueue` owns a single pending request slot and host -// request state owns any segment access until task-side poll completes it. -unsafe impl rdif_block::IQueue for SdmmcBlockQueue -where - H: SdmmcBlockHost, -{ - fn id(&self) -> usize { - self.id - } - - fn info(&self) -> rdif_block::QueueInfo { - rdif_block::QueueInfo { - id: self.id, - device: rdif_block::DeviceInfo { - name: Some(self.name), - ..rdif_block::DeviceInfo::new(self.capacity_blocks, BLOCK_SIZE) - }, - limits: queue_limits(&self.config, self.dma.dma_mask()), - } - } - - fn submit_request( - &mut self, - request: rdif_block::Request<'_>, - ) -> Result { - self.submit_request_inner(request) - } - - fn poll_request( - &mut self, - request: rdif_block::RequestId, - ) -> Result { - self.poll_request_inner(request) - } -} - -struct SdmmcBlockIrqHandler -where - H: SdmmcBlockHost, -{ - handle: ::IrqHandle, - _marker: PhantomData, -} - -impl rdif_block::IrqHandler for SdmmcBlockIrqHandler -where - H: SdmmcBlockHost, -{ - fn handle_irq(&self) -> rdif_block::Event { - let host_event = self.handle.handle_irq(); - let mut event = rdif_block::Event::none(); - if let Some(queue_id) = block_queue_ready_from_host_event(&host_event) { - event.push_queue(queue_id); - } - event - } -} - -pub(crate) fn queue_limits(config: &SdmmcBlockConfig, dma_mask: u64) -> rdif_block::QueueLimits { - rdif_block::QueueLimits { - dma_mask, - dma_alignment: BLOCK_SIZE, - max_inflight: 1, - max_blocks_per_request: config.max_blocks_per_request, - max_segments: 1, - max_segment_size: config.max_segment_size, - supported_flags: rdif_block::RequestFlags::NONE, - supports_flush: false, - supports_discard: false, - supports_write_zeroes: false, - } -} - -pub(crate) fn device_info(config: &SdmmcBlockConfig) -> rdif_block::DeviceInfo { - rdif_block::DeviceInfo { - name: Some(config.name), - ..rdif_block::DeviceInfo::new(config.capacity_blocks, BLOCK_SIZE) - } -} - -pub(crate) fn block_addr_for_card( - block_id: u64, - high_capacity: bool, -) -> Result { - let block_id = - u32::try_from(block_id).map_err(|_| rdif_block::BlkError::InvalidBlockIndex(block_id))?; - if high_capacity { - Ok(block_id) - } else { - block_id - .checked_mul(BLOCK_SIZE as u32) - .ok_or(rdif_block::BlkError::InvalidBlockIndex(block_id as u64)) - } -} - -pub(crate) fn map_dev_err_to_blk_err(err: Error) -> rdif_block::BlkError { - match err { - Error::NoCard | Error::UnsupportedCommand | Error::CardLocked => { - rdif_block::BlkError::NotSupported - } - Error::Misaligned | Error::InvalidArgument => { - rdif_block::BlkError::Other("SD/MMC request is not block aligned") - } - _ => rdif_block::BlkError::Io, - } -} - -#[cfg(any(feature = "k230-sdhci", feature = "rockchip-sdhci"))] -impl SdmmcBlockHost for sdhci_host::Sdhci { - type Request = sdhci_host::BlockRequest; - type Slot = sdhci_host::BlockRequestSlot; - - fn submit_read_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - submit_sdhci_read_request(self, start_block, buffer, size, dma, slot, pending) - } - - fn submit_write_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - submit_sdhci_write_request(self, start_block, buffer, size, dma, slot, pending) - } - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - slot: &mut Self::Slot, - ) -> Result { - self.poll_block_request( - pending, - sdhci_host::RequestId::new(usize::from(request)), - slot, - ) - } - - fn request_id(request: &Self::Request) -> BlockRequestId { - BlockRequestId::new(usize::from(request.id())) - } -} - -#[cfg(any(feature = "k230-sdhci", feature = "rockchip-sdhci"))] -pub(crate) fn submit_sdhci_read_request( - host: &mut sdhci_host::Sdhci, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut sdhci_host::BlockRequestSlot, - pending: &mut Option, -) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match host.submit_read_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => host - .submit_read_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)?, - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) -} - -#[cfg(any(feature = "k230-sdhci", feature = "rockchip-sdhci"))] -pub(crate) fn submit_sdhci_write_request( - host: &mut sdhci_host::Sdhci, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut sdhci_host::BlockRequestSlot, - pending: &mut Option, -) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match host.submit_write_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => host - .submit_write_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)?, - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) -} - -#[cfg(feature = "rockchip-dwmmc")] -impl SdmmcBlockHost for dwmmc_host::DwMmc { - type Request = dwmmc_host::BlockRequest; - type Slot = dwmmc_host::BlockRequestSlot; - - fn submit_read_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match self.submit_read_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => self - .submit_read_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)?, - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) - } - - fn submit_write_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match self.submit_write_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => self - .submit_write_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)?, - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) - } - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - slot: &mut Self::Slot, - ) -> Result { - self.poll_block_request( - pending, - dwmmc_host::RequestId::new(usize::from(request)), - slot, - ) - } - - fn request_id(request: &Self::Request) -> BlockRequestId { - BlockRequestId::new(usize::from(request.id())) - } -} - -#[cfg(feature = "phytium-mci")] -impl SdmmcBlockHost for phytium_mci_host::PhytiumMci { - type Request = phytium_mci_host::BlockRequest; - type Slot = phytium_mci_host::BlockRequestSlot; - - fn submit_read_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match self.submit_read_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => { - warn!( - "phytium-mci: DMA read unavailable ({:?}); falling back to FIFO", - err - ); - self.submit_read_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)? - } - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) - } - - fn submit_write_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match self.submit_write_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => { - warn!( - "phytium-mci: DMA write unavailable ({:?}); falling back to FIFO", - err - ); - self.submit_write_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)? - } - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) - } - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - slot: &mut Self::Slot, - ) -> Result { - self.poll_block_request( - pending, - phytium_mci_host::RequestId::new(usize::from(request)), - slot, - ) - } - - fn request_id(request: &Self::Request) -> BlockRequestId { - BlockRequestId::new(usize::from(request.id())) - } -} - -fn transfer_mode_for_dma(dma: Option<&DeviceDma>) -> sdmmc_protocol::BlockTransferMode { - match dma { - Some(_) => sdmmc_protocol::BlockTransferMode::Dma, - None => sdmmc_protocol::BlockTransferMode::Fifo, - } -} - -fn transfer_dma(use_dma: bool, dma: &DeviceDma) -> Option<&DeviceDma> { - use_dma.then_some(dma) -} - -fn can_fallback_to_fifo(err: Error) -> bool { - matches!( - err, - Error::UnsupportedCommand | Error::InvalidArgument | Error::Misaligned - ) -} - -#[cfg(test)] -mod tests { - use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - use sdmmc_protocol::{ - CommandResponsePoll, DataCommandPoll, - cmd::Command, - sdio::{ClockSpeed, HostEvent, HostEventKind}, - }; - - use super::*; - - #[test] - fn disabled_irq_policy_does_not_advertise_sources() { - let config = SdmmcBlockConfig::dma("test-sdmmc", 8, false); - - assert_eq!(queue_limits(&config, u32::MAX as u64).max_inflight, 1); - assert_eq!(device_info(&config).name, Some("test-sdmmc")); - } - - #[test] - fn fifo_config_limits_single_block_requests() { - let config = SdmmcBlockConfig::fifo("test-sdmmc", 8, true); - let limits = queue_limits(&config, u32::MAX as u64); - - assert_eq!(limits.max_blocks_per_request, 1); - assert_eq!(limits.max_segment_size, BLOCK_SIZE); - } - - #[test] - fn irq_policy_disabled_does_not_advertise_irq_sources() { - let raw = SharedDriver::new(SdioSdmmc::new(MockHost::default())); - let device = SdmmcBlockDevice::new(raw, SdmmcBlockConfig::dma("mock-sd", 8, false)); - - assert!(rdif_block::Interface::irq_sources(&device).is_empty()); - } - - #[test] - fn enabled_irq_handler_maps_host_event_to_queue_zero() { - let raw = SharedDriver::new(SdioSdmmc::new(MockHost::default())); - let mut device = SdmmcBlockDevice::new(raw, SdmmcBlockConfig::dma("mock-sd", 8, true)); - let handler = rdif_block::Interface::take_irq_handler(&mut device, 0).unwrap(); - - let event = handler.handle_irq(); - - assert!(event.queues.contains(0)); - assert!(!event.is_empty()); - } - - #[test] - fn poll_request_only_completes_matching_request_id() { - let raw = SharedDriver::new(SdioSdmmc::new(MockHost::default())); - let mut queue = SdmmcBlockQueue:: { - raw, - name: "mock-sd", - capacity_blocks: 8, - config: SdmmcBlockConfig::dma("mock-sd", 8, false), - id: 0, - dma: axklib::dma::device_with_mask(u32::MAX as u64), - slot: MockSlot, - pending: Some(MockRequest { - id: BlockRequestId::new(7), - }), - completed: Vec::new(), - }; - - assert_eq!( - queue.poll_request_inner(rdif_block::RequestId::new(8)), - Ok(rdif_block::RequestStatus::Pending) - ); - assert_eq!( - queue.poll_request_inner(rdif_block::RequestId::new(7)), - Ok(rdif_block::RequestStatus::Complete) - ); - assert!(queue.pending.is_none()); - } - - #[derive(Clone, Default)] - struct MockIrqHandle; - - impl SdioIrqHandle for MockIrqHandle { - type Event = MockEvent; - - fn handle_irq(&self) -> Self::Event { - MockEvent(HostEventKind::TransferComplete) - } - } - - #[derive(Clone, Copy, Default)] - struct MockEvent(HostEventKind); - - impl HostEvent for MockEvent { - fn kind(&self) -> HostEventKind { - self.0 - } - } - - #[derive(Default)] - struct MockHost { - irq_enabled: AtomicBool, - next_id: AtomicUsize, - } - - #[derive(Default)] - struct MockSlot; - - struct MockRequest { - id: BlockRequestId, - } - - impl SdioHost for MockHost { - type Event = MockEvent; - type DataRequest<'a> = (); - - fn submit_command(&mut self, _cmd: &Command) -> Result<(), Error> { - Err(Error::UnsupportedCommand) - } - - fn poll_command_response(&mut self) -> Result { - Ok(CommandResponsePoll::Pending) - } - - fn submit_read_data<'a>( - &mut self, - _cmd: &Command, - _buf: &'a mut [u8], - _block_size: u32, - _block_count: u32, - ) -> Result, Error> { - Err(Error::UnsupportedCommand) - } - - fn submit_write_data<'a>( - &mut self, - _cmd: &Command, - _buf: &'a [u8], - _block_size: u32, - _block_count: u32, - ) -> Result, Error> { - Err(Error::UnsupportedCommand) - } - - fn poll_data_request<'a>( - &mut self, - _request: &mut Self::DataRequest<'a>, - ) -> Result { - Err(Error::UnsupportedCommand) - } - - fn set_bus_width(&mut self, _width: sdmmc_protocol::sdio::BusWidth) -> Result<(), Error> { - Ok(()) - } - - fn set_clock(&mut self, _speed: ClockSpeed) -> Result<(), Error> { - Ok(()) - } - - fn enable_completion_irq(&mut self) -> Result<(), Error> { - self.irq_enabled.store(true, Ordering::Release); - Ok(()) - } - - fn disable_completion_irq(&mut self) -> Result<(), Error> { - self.irq_enabled.store(false, Ordering::Release); - Ok(()) - } - } - - impl SdioIrqHost for MockHost { - type IrqHandle = MockIrqHandle; - - fn irq_handle(&self) -> Self::IrqHandle { - MockIrqHandle - } - - fn completion_irq_enabled(&self) -> bool { - self.irq_enabled.load(Ordering::Acquire) - } - } - - impl SdmmcBlockHost for MockHost { - type Request = MockRequest; - type Slot = MockSlot; - - fn submit_read_request( - &mut self, - _start_block: u32, - _buffer: NonNull, - _size: NonZeroUsize, - _dma: Option<&DeviceDma>, - _slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - self.submit_mock_request(pending) - } - - fn submit_write_request( - &mut self, - _start_block: u32, - _buffer: NonNull, - _size: NonZeroUsize, - _dma: Option<&DeviceDma>, - _slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - self.submit_mock_request(pending) - } - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - _slot: &mut Self::Slot, - ) -> Result { - match pending.as_ref() { - Some(active) if active.id == request => { - *pending = None; - Ok(BlockPoll::Complete) - } - Some(_) => Ok(BlockPoll::Pending), - None => Ok(BlockPoll::Complete), - } - } - - fn request_id(request: &Self::Request) -> BlockRequestId { - request.id - } - } - - impl MockHost { - fn submit_mock_request( - &self, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let id = BlockRequestId::new(self.next_id.fetch_add(1, Ordering::Relaxed)); - *pending = Some(MockRequest { id }); - Ok(id) - } - } -} diff --git a/drivers/blk/dwmmc-host/Cargo.toml b/drivers/blk/dwmmc-host/Cargo.toml index a08e4bb37f..3cd3b8ab4f 100644 --- a/drivers/blk/dwmmc-host/Cargo.toml +++ b/drivers/blk/dwmmc-host/Cargo.toml @@ -10,13 +10,14 @@ keywords = ["sd", "mmc", "dwmmc", "embedded", "no_std"] categories = ["embedded", "no-std", "hardware-support"] [dependencies] -sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio"] } +sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio", "rdif"] } embedded-hal = "1" bitfield-struct = "0.11" volatile = { version = "0.6", features = ["derive"] } log.workspace = true dma-api.workspace = true mmio-api.workspace = true +rdif-block.workspace = true [features] default = [] diff --git a/drivers/blk/dwmmc-host/src/lib.rs b/drivers/blk/dwmmc-host/src/lib.rs index b68f16d2ee..ed31ba46b5 100644 --- a/drivers/blk/dwmmc-host/src/lib.rs +++ b/drivers/blk/dwmmc-host/src/lib.rs @@ -60,6 +60,7 @@ use core::{marker::PhantomData, num::NonZeroUsize, ptr::NonNull}; mod command; mod dma; mod host; +pub mod rdif; mod regs; pub use sdmmc_protocol::block::{ diff --git a/drivers/blk/dwmmc-host/src/rdif.rs b/drivers/blk/dwmmc-host/src/rdif.rs new file mode 100644 index 0000000000..fd4425f989 --- /dev/null +++ b/drivers/blk/dwmmc-host/src/rdif.rs @@ -0,0 +1,175 @@ +//! RDIF block-device adapter for [`DwMmc`]. + +use core::{num::NonZeroUsize, ptr::NonNull}; + +use dma_api::DeviceDma; +pub use protocol_rdif::{BlockConfig, BlockDevice, BlockQueue}; +pub use rdif_block::{ + BlkError, IQueue, Interface, Request, RequestId as RdifRequestId, RequestStatus, +}; +use sdmmc_protocol::{BlockPoll, BlockRequestId, Error, rdif as protocol_rdif, sdio::SdioSdmmc}; + +use crate::{BlockRequest, BlockRequestSlot, DwMmc, RequestId}; + +pub fn device(card: SdioSdmmc, config: BlockConfig) -> BlockDevice { + BlockDevice::new(card, config) +} + +pub fn dma_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, + dma: DeviceDma, +) -> BlockConfig { + BlockConfig::dma(name, capacity_blocks, irq_driven, dma) + .with_max_blocks_per_request(1024) + .with_max_segment_size(1024 * protocol_rdif::BLOCK_SIZE) +} + +pub const fn fifo_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, +) -> BlockConfig { + BlockConfig::fifo(name, capacity_blocks, irq_driven) +} + +impl protocol_rdif::BlockHost for DwMmc { + type Request = BlockRequest; + type Slot = BlockRequestSlot; + + fn submit_read_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_request( + self, + Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction: Direction::Read, + }, + ) + } + + fn submit_write_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_request( + self, + Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction: Direction::Write, + }, + ) + } + + fn poll_block_request( + &mut self, + pending: &mut Option, + request: BlockRequestId, + slot: &mut Self::Slot, + ) -> Result { + self.poll_block_request(pending, RequestId::new(usize::from(request)), slot) + } + + fn request_id(request: &Self::Request) -> BlockRequestId { + BlockRequestId::new(usize::from(request.id())) + } +} + +#[derive(Clone, Copy)] +enum Direction { + Read, + Write, +} + +struct Submission<'a> { + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&'a DeviceDma>, + slot: &'a mut BlockRequestSlot, + pending: &'a mut Option, + direction: Direction, +} + +fn submit_request( + host: &mut DwMmc, + submission: Submission<'_>, +) -> Result { + let Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction, + } = submission; + if pending.is_some() { + return Err(BlkError::Retry); + } + let request = match submit_blocks(host, start_block, buffer, size, dma, slot, direction) { + Ok(request) => request, + Err(err) if dma.is_some() && protocol_rdif::can_fallback_to_fifo(err) => { + submit_blocks(host, start_block, buffer, size, None, slot, direction) + .map_err(protocol_rdif::map_dev_err_to_blk_err)? + } + Err(err) => return Err(protocol_rdif::map_dev_err_to_blk_err(err)), + }; + let id = request.id(); + *pending = Some(request); + Ok(BlockRequestId::new(usize::from(id))) +} + +fn submit_blocks( + host: &mut DwMmc, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut BlockRequestSlot, + direction: Direction, +) -> Result { + let mode = protocol_rdif::transfer_mode_for_dma(dma); + match direction { + Direction::Read => host.submit_read_blocks(start_block, buffer, size, dma, mode, slot), + Direction::Write => host.submit_write_blocks(start_block, buffer, size, dma, mode, slot), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fifo_config_keeps_one_block_limits() { + let config = fifo_config("dwmmc", 16, true); + let limits = protocol_rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, 1); + assert_eq!(limits.max_segment_size, protocol_rdif::BLOCK_SIZE); + assert!(!config.uses_dma()); + } +} diff --git a/drivers/blk/phytium-mci-host/Cargo.toml b/drivers/blk/phytium-mci-host/Cargo.toml index 3a36d30b6f..dd1448cb0a 100644 --- a/drivers/blk/phytium-mci-host/Cargo.toml +++ b/drivers/blk/phytium-mci-host/Cargo.toml @@ -10,12 +10,13 @@ keywords = ["sd", "mmc", "phytium", "embedded", "no_std"] categories = ["embedded", "no-std", "hardware-support"] [dependencies] -sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio"] } +sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio", "rdif"] } bitfield-struct = "0.11" volatile = { version = "0.6", features = ["derive"] } log.workspace = true dma-api.workspace = true mmio-api.workspace = true +rdif-block.workspace = true [features] default = [] diff --git a/drivers/blk/phytium-mci-host/src/lib.rs b/drivers/blk/phytium-mci-host/src/lib.rs index 5d20eb1565..ea57d39180 100644 --- a/drivers/blk/phytium-mci-host/src/lib.rs +++ b/drivers/blk/phytium-mci-host/src/lib.rs @@ -25,6 +25,7 @@ use core::{marker::PhantomData, ptr::NonNull}; mod command; mod dma; mod host; +pub mod rdif; mod regs; mod timing; diff --git a/drivers/blk/phytium-mci-host/src/rdif.rs b/drivers/blk/phytium-mci-host/src/rdif.rs new file mode 100644 index 0000000000..cff01a7d45 --- /dev/null +++ b/drivers/blk/phytium-mci-host/src/rdif.rs @@ -0,0 +1,175 @@ +//! RDIF block-device adapter for [`PhytiumMci`]. + +use core::{num::NonZeroUsize, ptr::NonNull}; + +use dma_api::DeviceDma; +pub use protocol_rdif::{BlockConfig, BlockDevice, BlockQueue}; +pub use rdif_block::{ + BlkError, IQueue, Interface, Request, RequestId as RdifRequestId, RequestStatus, +}; +use sdmmc_protocol::{BlockPoll, BlockRequestId, Error, rdif as protocol_rdif, sdio::SdioSdmmc}; + +use crate::{BlockRequest, BlockRequestSlot, PhytiumMci, RequestId}; + +pub fn device(card: SdioSdmmc, config: BlockConfig) -> BlockDevice { + BlockDevice::new(card, config) +} + +pub fn dma_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, + dma: DeviceDma, +) -> BlockConfig { + BlockConfig::dma(name, capacity_blocks, irq_driven, dma) + .with_max_blocks_per_request(1024) + .with_max_segment_size(1024 * protocol_rdif::BLOCK_SIZE) +} + +pub const fn fifo_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, +) -> BlockConfig { + BlockConfig::fifo(name, capacity_blocks, irq_driven) +} + +impl protocol_rdif::BlockHost for PhytiumMci { + type Request = BlockRequest; + type Slot = BlockRequestSlot; + + fn submit_read_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_request( + self, + Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction: Direction::Read, + }, + ) + } + + fn submit_write_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_request( + self, + Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction: Direction::Write, + }, + ) + } + + fn poll_block_request( + &mut self, + pending: &mut Option, + request: BlockRequestId, + slot: &mut Self::Slot, + ) -> Result { + self.poll_block_request(pending, RequestId::new(usize::from(request)), slot) + } + + fn request_id(request: &Self::Request) -> BlockRequestId { + BlockRequestId::new(usize::from(request.id())) + } +} + +#[derive(Clone, Copy)] +enum Direction { + Read, + Write, +} + +struct Submission<'a> { + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&'a DeviceDma>, + slot: &'a mut BlockRequestSlot, + pending: &'a mut Option, + direction: Direction, +} + +fn submit_request( + host: &mut PhytiumMci, + submission: Submission<'_>, +) -> Result { + let Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction, + } = submission; + if pending.is_some() { + return Err(BlkError::Retry); + } + let request = match submit_blocks(host, start_block, buffer, size, dma, slot, direction) { + Ok(request) => request, + Err(err) if dma.is_some() && protocol_rdif::can_fallback_to_fifo(err) => { + submit_blocks(host, start_block, buffer, size, None, slot, direction) + .map_err(protocol_rdif::map_dev_err_to_blk_err)? + } + Err(err) => return Err(protocol_rdif::map_dev_err_to_blk_err(err)), + }; + let id = request.id(); + *pending = Some(request); + Ok(BlockRequestId::new(usize::from(id))) +} + +fn submit_blocks( + host: &mut PhytiumMci, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut BlockRequestSlot, + direction: Direction, +) -> Result { + let mode = protocol_rdif::transfer_mode_for_dma(dma); + match direction { + Direction::Read => host.submit_read_blocks(start_block, buffer, size, dma, mode, slot), + Direction::Write => host.submit_write_blocks(start_block, buffer, size, dma, mode, slot), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fifo_config_keeps_one_block_limits() { + let config = fifo_config("phytium-mci", 16, true); + let limits = protocol_rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, 1); + assert_eq!(limits.max_segment_size, protocol_rdif::BLOCK_SIZE); + assert!(!config.uses_dma()); + } +} diff --git a/drivers/blk/sdhci-host/Cargo.toml b/drivers/blk/sdhci-host/Cargo.toml index 9803fb13d4..ba2dbcb90d 100644 --- a/drivers/blk/sdhci-host/Cargo.toml +++ b/drivers/blk/sdhci-host/Cargo.toml @@ -10,11 +10,12 @@ keywords = ["sd", "sdhci", "embedded", "no_std"] categories = ["embedded", "no-std", "hardware-support"] [dependencies] -sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio"] } +sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio", "rdif"] } embedded-hal = "1" log.workspace = true dma-api.workspace = true mmio-api.workspace = true +rdif-block.workspace = true [features] default = [] diff --git a/drivers/blk/sdhci-host/src/dma.rs b/drivers/blk/sdhci-host/src/dma.rs index 74a15fce62..6ccf54a054 100644 --- a/drivers/blk/sdhci-host/src/dma.rs +++ b/drivers/blk/sdhci-host/src/dma.rs @@ -75,6 +75,10 @@ const ADMA2_MAX_PER_DESC: usize = 65_528; // 64 KiB - 8B, multiple of 8 pub const ADMA2_DESC_COUNT: usize = 16; pub const ADMA2_DESC_ALIGN: usize = 64; const BLOCK_SIZE: usize = 512; +pub const ADMA2_MAX_TRANSFER_SIZE: usize = + (ADMA2_DESC_COUNT * ADMA2_MAX_PER_DESC / BLOCK_SIZE) * BLOCK_SIZE; +pub const ADMA2_MAX_BLOCKS: u32 = (ADMA2_MAX_TRANSFER_SIZE / BLOCK_SIZE) as u32; +const DWC_MSHC_ADMA_BOUNDARY: u64 = 128 * 1024 * 1024; pub type RequestId = BlockRequestId; @@ -255,7 +259,10 @@ pub(crate) fn build_descriptors( if written >= ADMA2_DESC_COUNT { return Err(Error::Misaligned); } - let chunk = remaining.min(ADMA2_MAX_PER_DESC); + let boundary_room = DWC_MSHC_ADMA_BOUNDARY - ((base + offset) % DWC_MSHC_ADMA_BOUNDARY); + let chunk = remaining + .min(ADMA2_MAX_PER_DESC) + .min(boundary_room as usize); let is_last = chunk == remaining; let mut attr = ADMA2_ATTR_VALID | ADMA2_ATTR_ACT_TRAN; if is_last { @@ -1213,6 +1220,21 @@ mod tests { assert_eq!(table[1].address, 0x2000_0000 + ADMA2_MAX_PER_DESC as u32); } + #[test] + fn splits_at_dwcmshc_128m_boundary() { + let mut table = empty_table(); + let base = DWC_MSHC_ADMA_BOUNDARY - 1024; + let n = build_descriptors(&mut table, base, 4096, Phase::DataRead).unwrap(); + + assert_eq!(n, 2); + assert_eq!(table[0].length, 1024); + assert_eq!(table[0].address, base as u32); + assert!(table[0].attr & ADMA2_ATTR_END == 0); + assert_eq!(table[1].length, 3072); + assert_eq!(table[1].address, DWC_MSHC_ADMA_BOUNDARY as u32); + assert!(table[1].attr & ADMA2_ATTR_END != 0); + } + #[test] fn rejects_64bit_bus_address() { let mut table = empty_table(); diff --git a/drivers/blk/sdhci-host/src/lib.rs b/drivers/blk/sdhci-host/src/lib.rs index 1e6e9ece52..7f167dbbb0 100644 --- a/drivers/blk/sdhci-host/src/lib.rs +++ b/drivers/blk/sdhci-host/src/lib.rs @@ -7,11 +7,11 @@ //! # Scope //! //! - **Implemented**: PIO transfers, **ADMA2 (32-bit) transfers**, 1-bit / -//! 4-bit bus, default-speed and high-speed clocking, 32-bit response +//! 4-bit / 8-bit bus, default-speed and high-speed clocking, 32-bit response //! slots, 136-bit R2 reconstruction, software reset / clock setup. -//! - **Out of scope (for now)**: 64-bit ADMA2, 8-bit eMMC bus, HS200 / -//! SDR50 / SDR104 clocking, tuning (CMD19 / CMD21), eMMC-specific -//! commands. 1.8 V signaling is wired up at the register level but is +//! - **Out of scope (for now)**: 64-bit ADMA2, HS200 / SDR50 / SDR104 +//! clocking, tuning (CMD19 / CMD21), eMMC-specific commands beyond normal +//! block I/O. 1.8 V signaling is wired up at the register level but is //! gated behind [`Sdhci::enable_1v8_signaling`] — platforms that haven't //! plumbed the IO-rail regulator MUST leave it off so the protocol //! layer falls back instead of corrupting transfers. @@ -76,9 +76,13 @@ use core::{marker::PhantomData, num::NonZeroUsize, ptr::NonNull}; mod command; mod dma; mod host; +pub mod rdif; mod regs; -pub use dma::{ADMA2_DESC_ALIGN, ADMA2_DESC_COUNT, BlockRequest, BlockRequestSlot, RequestId}; +pub use dma::{ + ADMA2_DESC_ALIGN, ADMA2_DESC_COUNT, ADMA2_MAX_BLOCKS, ADMA2_MAX_TRANSFER_SIZE, BlockRequest, + BlockRequestSlot, RequestId, +}; pub use host::{HostClock, Sdhci}; pub use sdmmc_protocol::block::{ BlockBufferConfig, BlockPoll, BlockRequestId, BlockTransferDirection, BlockTransferMode, @@ -106,6 +110,10 @@ pub enum Event { CommandComplete, /// A data transfer has completed. TransferComplete, + /// Receive-side FIFO data is ready. + ReceiveReady, + /// Transmit-side FIFO space is ready. + TransmitReady, /// One or more error bits are pending. Error { normal: u16, error: u16 }, /// Status bits are pending but do not map to a high-level event yet. @@ -211,11 +219,7 @@ impl SdioHost for Sdhci { match width { BusWidth::Bit1 => {} BusWidth::Bit4 => ctrl |= HOST_CTRL1_4BIT, - // 8-bit is eMMC territory and is intentionally not part of the - // MVP — surface it as Unsupported so the protocol layer can - // refuse cleanly instead of silently writing the bit and - // misconfiguring the bus. - BusWidth::Bit8 => return Err(Error::UnsupportedCommand), + BusWidth::Bit8 => ctrl |= HOST_CTRL1_8BIT, // Future BusWidth variants are not supported by this controller. _ => return Err(Error::UnsupportedCommand), } @@ -224,17 +228,25 @@ impl SdioHost for Sdhci { } fn set_clock(&mut self, speed: ClockSpeed) -> Result<(), Error> { - let target_hz = match speed { - ClockSpeed::Identification => 400_000, - ClockSpeed::Default | ClockSpeed::Sdr12 => 25_000_000, - ClockSpeed::HighSpeed | ClockSpeed::Sdr25 => 50_000_000, - ClockSpeed::Sdr50 | ClockSpeed::Ddr50 => 50_000_000, - ClockSpeed::Sdr104 => 104_000_000, - ClockSpeed::Hs200 => 200_000_000, + let (target_hz, uhs_mode) = match speed { + ClockSpeed::Identification => (400_000, HOST_CTRL2_UHS_SDR12), + ClockSpeed::Default | ClockSpeed::Sdr12 => (25_000_000, HOST_CTRL2_UHS_SDR12), + ClockSpeed::HighSpeed | ClockSpeed::Sdr25 => (50_000_000, HOST_CTRL2_UHS_SDR25), + ClockSpeed::Sdr50 => (50_000_000, HOST_CTRL2_UHS_SDR50), + ClockSpeed::Ddr50 => (50_000_000, HOST_CTRL2_UHS_DDR50), + ClockSpeed::Sdr104 => (104_000_000, HOST_CTRL2_UHS_SDR104), + ClockSpeed::Hs200 => (200_000_000, HOST_CTRL2_UHS_SDR104), // Future ClockSpeed variants are not supported by this controller. _ => return Err(Error::UnsupportedCommand), }; + // Match Linux's SDHCI/DWCMSHC UHS signaling selection: even legacy + // MMC HighSpeed maps to the SDR25 bus-speed mode on controllers that + // interpret HOST_CONTROL2.UHS_MODE_SELECT. + let mut ctrl2 = self.read_u16(REG_HOST_CONTROL2); + ctrl2 = (ctrl2 & !HOST_CTRL2_UHS_MODE_MASK) | uhs_mode; + self.write_u16(REG_HOST_CONTROL2, ctrl2); + // Toggle the High-Speed Enable bit in HOST_CONTROL1 alongside the // divider change so the controller pipelines reflect the new // timing window. @@ -519,6 +531,10 @@ pub(crate) fn event_from_status(normal: u16, error: u16) -> Event { Event::CommandComplete } else if normal & NORMAL_INT_XFER_COMPLETE != 0 { Event::TransferComplete + } else if normal & NORMAL_INT_BUFFER_READ_READY != 0 { + Event::ReceiveReady + } else if normal & NORMAL_INT_BUFFER_WRITE_READY != 0 { + Event::TransmitReady } else if normal != 0 || error != 0 { Event::Other { normal, error } } else { @@ -532,6 +548,8 @@ impl HostEvent for Event { Event::None => HostEventKind::None, Event::CommandComplete => HostEventKind::CommandComplete, Event::TransferComplete => HostEventKind::TransferComplete, + Event::ReceiveReady => HostEventKind::ReceiveReady, + Event::TransmitReady => HostEventKind::TransmitReady, Event::Error { .. } => HostEventKind::Error, Event::Other { .. } => HostEventKind::Other, } @@ -540,14 +558,18 @@ impl HostEvent for Event { fn source(&self) -> HostEventSource { match self { Event::CommandComplete => HostEventSource::Command, - Event::TransferComplete => HostEventSource::Data, + Event::TransferComplete | Event::ReceiveReady | Event::TransmitReady => { + HostEventSource::Data + } Event::None | Event::Error { .. } | Event::Other { .. } => HostEventSource::Controller, } } fn queue_id(&self) -> Option { match self { - Event::TransferComplete => Some(BlockRequestId::new(0)), + Event::TransferComplete | Event::ReceiveReady | Event::TransmitReady => { + Some(BlockRequestId::new(0)) + } Event::None | Event::CommandComplete | Event::Error { .. } | Event::Other { .. } => { None } diff --git a/drivers/blk/sdhci-host/src/rdif.rs b/drivers/blk/sdhci-host/src/rdif.rs new file mode 100644 index 0000000000..4b1423747f --- /dev/null +++ b/drivers/blk/sdhci-host/src/rdif.rs @@ -0,0 +1,233 @@ +//! RDIF block-device adapter for [`Sdhci`]. + +use core::{num::NonZeroUsize, ptr::NonNull}; + +use dma_api::DeviceDma; +pub use protocol_rdif::{BlockConfig, BlockDevice, BlockQueue}; +pub use rdif_block::{ + BlkError, IQueue, Interface, Request, RequestId as RdifRequestId, RequestStatus, +}; +use sdmmc_protocol::{BlockPoll, BlockRequestId, Error, rdif as protocol_rdif, sdio::SdioSdmmc}; + +use crate::{ + ADMA2_MAX_BLOCKS, ADMA2_MAX_TRANSFER_SIZE, BlockRequest, BlockRequestSlot, RequestId, Sdhci, +}; + +pub fn device(card: SdioSdmmc, config: BlockConfig) -> BlockDevice { + BlockDevice::new(card, config) +} + +pub fn dma_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, + dma: DeviceDma, +) -> BlockConfig { + BlockConfig::dma(name, capacity_blocks, irq_driven, dma) + .with_max_blocks_per_request(ADMA2_MAX_BLOCKS) + .with_max_segment_size(ADMA2_MAX_TRANSFER_SIZE) +} + +pub const fn fifo_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, +) -> BlockConfig { + BlockConfig::fifo(name, capacity_blocks, irq_driven) +} + +impl protocol_rdif::BlockHost for Sdhci { + type Request = BlockRequest; + type Slot = BlockRequestSlot; + + fn submit_read_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_request( + self, + Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction: Direction::Read, + }, + ) + } + + fn submit_write_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_request( + self, + Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction: Direction::Write, + }, + ) + } + + fn poll_block_request( + &mut self, + pending: &mut Option, + request: BlockRequestId, + slot: &mut Self::Slot, + ) -> Result { + self.poll_block_request(pending, RequestId::new(usize::from(request)), slot) + } + + fn request_id(request: &Self::Request) -> BlockRequestId { + BlockRequestId::new(usize::from(request.id())) + } +} + +#[derive(Clone, Copy)] +enum Direction { + Read, + Write, +} + +struct Submission<'a> { + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&'a DeviceDma>, + slot: &'a mut BlockRequestSlot, + pending: &'a mut Option, + direction: Direction, +} + +fn submit_request( + host: &mut Sdhci, + submission: Submission<'_>, +) -> Result { + let Submission { + start_block, + buffer, + size, + dma, + slot, + pending, + direction, + } = submission; + if pending.is_some() { + return Err(BlkError::Retry); + } + let request = match submit_blocks(host, start_block, buffer, size, dma, slot, direction) { + Ok(request) => request, + Err(err) if dma.is_some() && protocol_rdif::can_fallback_to_fifo(err) => { + submit_blocks(host, start_block, buffer, size, None, slot, direction) + .map_err(protocol_rdif::map_dev_err_to_blk_err)? + } + Err(err) => return Err(protocol_rdif::map_dev_err_to_blk_err(err)), + }; + let id = request.id(); + *pending = Some(request); + Ok(BlockRequestId::new(usize::from(id))) +} + +fn submit_blocks( + host: &mut Sdhci, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut BlockRequestSlot, + direction: Direction, +) -> Result { + let mode = protocol_rdif::transfer_mode_for_dma(dma); + match direction { + Direction::Read => host.submit_read_blocks(start_block, buffer, size, dma, mode, slot), + Direction::Write => host.submit_write_blocks(start_block, buffer, size, dma, mode, slot), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fifo_config_keeps_one_block_limits() { + let config = fifo_config("sdhci", 16, true); + let limits = protocol_rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, 1); + assert_eq!(limits.max_segment_size, protocol_rdif::BLOCK_SIZE); + assert!(!config.uses_dma()); + } + + #[test] + fn dma_config_advertises_adma_window() { + let config = dma_config( + "sdhci", + 16, + true, + dma_api::DeviceDma::new(u32::MAX as u64, &TEST_DMA), + ); + let limits = protocol_rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, ADMA2_MAX_BLOCKS); + assert_eq!(limits.max_segment_size, ADMA2_MAX_TRANSFER_SIZE); + assert!(config.uses_dma()); + } + + struct TestDma; + static TEST_DMA: TestDma = TestDma; + + impl dma_api::DmaOp for TestDma { + fn page_size(&self) -> usize { + protocol_rdif::BLOCK_SIZE + } + + unsafe fn alloc_contiguous( + &self, + _constraints: dma_api::DmaConstraints, + _layout: core::alloc::Layout, + ) -> Option { + None + } + + unsafe fn dealloc_contiguous(&self, _handle: dma_api::DmaAllocHandle) {} + + unsafe fn alloc_coherent( + &self, + _constraints: dma_api::DmaConstraints, + _layout: core::alloc::Layout, + ) -> Option { + None + } + + unsafe fn dealloc_coherent(&self, _handle: dma_api::DmaAllocHandle) {} + + unsafe fn map_streaming( + &self, + _constraints: dma_api::DmaConstraints, + _addr: core::ptr::NonNull, + _size: core::num::NonZeroUsize, + _direction: dma_api::DmaDirection, + ) -> Result { + Err(dma_api::DmaError::NoMemory) + } + + unsafe fn unmap_streaming(&self, _handle: dma_api::DmaMapHandle) {} + } +} diff --git a/drivers/blk/sdmmc-protocol/Cargo.toml b/drivers/blk/sdmmc-protocol/Cargo.toml index 79743bb339..6ff739eead 100644 --- a/drivers/blk/sdmmc-protocol/Cargo.toml +++ b/drivers/blk/sdmmc-protocol/Cargo.toml @@ -13,8 +13,10 @@ categories = ["embedded", "no-std", "hardware-support"] bitflags = "2" embedded-hal = "1" log.workspace = true +rdif-block = { workspace = true, optional = true } [features] default = ["spi"] spi = [] sdio = [] +rdif = ["sdio", "dep:rdif-block"] diff --git a/drivers/blk/sdmmc-protocol/src/lib.rs b/drivers/blk/sdmmc-protocol/src/lib.rs index 4971910d99..f46ada5f44 100644 --- a/drivers/blk/sdmmc-protocol/src/lib.rs +++ b/drivers/blk/sdmmc-protocol/src/lib.rs @@ -81,6 +81,9 @@ #![no_std] +#[cfg(feature = "rdif")] +extern crate alloc; + pub mod block; pub mod cmd; mod common; @@ -94,6 +97,9 @@ pub mod spi; #[cfg(feature = "sdio")] pub mod sdio; +#[cfg(feature = "rdif")] +pub mod rdif; + pub use block::{ BlockBufferConfig, BlockPoll, BlockRequestId, BlockTransferDirection, BlockTransferMode, BlockTransferState, CommandPoll, CommandResponsePoll, DataCommandDirection, DataCommandPoll, diff --git a/drivers/blk/sdmmc-protocol/src/rdif.rs b/drivers/blk/sdmmc-protocol/src/rdif.rs new file mode 100644 index 0000000000..1922221f80 --- /dev/null +++ b/drivers/blk/sdmmc-protocol/src/rdif.rs @@ -0,0 +1,813 @@ +//! RDIF block-device bridge for SDIO-backed SD/MMC hosts. +//! +//! This module owns the reusable queue/runtime-independent part of adapting a +//! [`crate::sdio::SdioSdmmc`] card to [`rdif_block`]. Host crates provide the +//! small controller-specific [`BlockHost`] impl that submits and polls one +//! block request. + +use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; +use core::{ + cell::UnsafeCell, + marker::PhantomData, + num::NonZeroUsize, + ptr::NonNull, + sync::atomic::{AtomicBool, Ordering}, +}; + +use log::warn; +use rdif_block::dma_api::DeviceDma; +pub use rdif_block::{BlkError, IQueue, Interface, Request, RequestId, RequestStatus, dma_api}; + +use crate::{ + BlockPoll, BlockRequestId, BlockTransferMode, Error, + sdio::{SdioHost, SdioIrqHandle, SdioIrqHost, SdioSdmmc, block_queue_ready_from_host_event}, +}; + +pub const BLOCK_SIZE: usize = 512; +pub const DEFAULT_DMA_MASK: u64 = u32::MAX as u64; +pub const DEFAULT_DMA_MAX_BLOCKS_PER_REQUEST: u32 = u16::MAX as u32 + 1; + +#[derive(Clone)] +pub struct BlockConfig { + pub name: &'static str, + pub capacity_blocks: u64, + pub dma_mask: u64, + pub max_blocks_per_request: u32, + pub max_segment_size: usize, + pub irq_driven: bool, + pub dma: Option, +} + +impl BlockConfig { + pub fn dma(name: &'static str, capacity_blocks: u64, irq_driven: bool, dma: DeviceDma) -> Self { + let dma_mask = dma.dma_mask(); + Self { + name, + capacity_blocks, + dma_mask, + max_blocks_per_request: DEFAULT_DMA_MAX_BLOCKS_PER_REQUEST, + max_segment_size: usize::MAX, + irq_driven, + dma: Some(dma), + } + } + + pub const fn fifo(name: &'static str, capacity_blocks: u64, irq_driven: bool) -> Self { + Self { + name, + capacity_blocks, + dma_mask: DEFAULT_DMA_MASK, + max_blocks_per_request: 1, + max_segment_size: BLOCK_SIZE, + irq_driven, + dma: None, + } + } + + pub fn with_dma_mask(mut self, dma_mask: u64) -> Self { + self.dma_mask = dma_mask; + self + } + + pub fn with_max_blocks_per_request(mut self, max_blocks_per_request: u32) -> Self { + self.max_blocks_per_request = max_blocks_per_request; + self + } + + pub fn with_max_segment_size(mut self, max_segment_size: usize) -> Self { + self.max_segment_size = max_segment_size; + self + } + + pub fn with_irq_driven(mut self, irq_driven: bool) -> Self { + self.irq_driven = irq_driven; + self + } + + pub fn with_dma(mut self, dma: DeviceDma) -> Self { + self.dma_mask = dma.dma_mask(); + self.dma = Some(dma); + self + } + + pub const fn uses_dma(&self) -> bool { + self.dma.is_some() + } +} + +pub trait BlockHost: SdioIrqHost + Send + Sync + 'static { + type Request: Send + 'static; + type Slot: Default + Send + 'static; + + fn submit_read_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result; + + fn submit_write_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result; + + fn poll_block_request( + &mut self, + pending: &mut Option, + request: BlockRequestId, + slot: &mut Self::Slot, + ) -> Result; + + fn request_id(request: &Self::Request) -> BlockRequestId; +} + +pub struct BlockDevice +where + H: BlockHost, +{ + raw: Option>>, + irq_handle: ::IrqHandle, + config: BlockConfig, + irq_enabled: AtomicBool, + queue_created: bool, + irq_handler_taken: bool, +} + +impl BlockDevice +where + H: BlockHost, +{ + pub fn new(card: SdioSdmmc, config: BlockConfig) -> Self { + let raw = SharedCore::new(card); + let irq_handle = raw.with_mut(|raw| raw.host().irq_handle()); + Self { + raw: Some(raw), + irq_handle, + config, + irq_enabled: AtomicBool::new(false), + queue_created: false, + irq_handler_taken: false, + } + } + + pub fn config(&self) -> &BlockConfig { + &self.config + } + + fn queue_limits_with_mask(&self, dma_mask: u64) -> rdif_block::QueueLimits { + queue_limits(&self.config, dma_mask) + } +} + +impl rdif_block::DriverGeneric for BlockDevice +where + H: BlockHost, +{ + fn name(&self) -> &str { + self.config.name + } +} + +impl Interface for BlockDevice +where + H: BlockHost, +{ + fn device_info(&self) -> rdif_block::DeviceInfo { + device_info(&self.config) + } + + fn queue_limits(&self) -> rdif_block::QueueLimits { + self.queue_limits_with_mask(self.config.dma_mask) + } + + fn create_queue(&mut self) -> Option> { + if self.queue_created { + return None; + } + self.raw.clone().map(|dev| { + self.queue_created = true; + Box::new(BlockQueue::::new(dev, self.config.clone(), 0)) as _ + }) + } + + fn enable_irq(&self) { + if !self.config.irq_driven { + self.irq_enabled.store(false, Ordering::Release); + return; + } + if let Some(raw) = &self.raw { + let mut enabled = false; + raw.with_mut(|raw| { + if let Err(err) = SdioHost::enable_completion_irq(raw.host_mut()) { + warn!( + "{}: enable completion IRQ failed: {:?}", + self.config.name, err + ); + return; + } + enabled = raw.host().completion_irq_enabled(); + }); + self.irq_enabled.store(enabled, Ordering::Release); + } + } + + fn disable_irq(&self) { + if let Some(raw) = &self.raw { + raw.with_mut(|raw| { + if let Err(err) = SdioHost::disable_completion_irq(raw.host_mut()) { + warn!( + "{}: disable completion IRQ failed: {:?}", + self.config.name, err + ); + } + }); + } + self.irq_enabled.store(false, Ordering::Release); + } + + fn is_irq_enabled(&self) -> bool { + self.irq_enabled.load(Ordering::Acquire) + } + + fn irq_sources(&self) -> rdif_block::IrqSourceList { + if !self.config.irq_driven { + return Vec::new(); + } + vec![rdif_block::IrqSourceInfo::legacy( + rdif_block::IdList::from_bits(1), + )] + } + + fn take_irq_handler(&mut self, source_id: usize) -> Option> { + if !self.config.irq_driven || source_id != 0 || self.irq_handler_taken { + return None; + } + self.irq_handler_taken = true; + Some(Box::new(BlockIrqHandler:: { + handle: self.irq_handle.clone(), + _marker: PhantomData, + })) + } +} + +pub struct BlockQueue +where + H: BlockHost, +{ + raw: SharedCore>, + config: BlockConfig, + id: usize, + slot: H::Slot, + pending: Option, + completed: Vec, +} + +impl BlockQueue +where + H: BlockHost, +{ + fn new(raw: SharedCore>, config: BlockConfig, id: usize) -> Self { + Self { + raw, + config, + id, + slot: H::Slot::default(), + pending: None, + completed: Vec::new(), + } + } + + fn queue_info(&self) -> rdif_block::QueueInfo { + rdif_block::IQueue::info(self) + } + + fn submit_request_inner(&mut self, request: Request<'_>) -> Result { + rdif_block::validate_request(self.queue_info(), &request)?; + self.reap_pending_request()?; + let raw = self.raw.clone(); + raw.with_mut(|raw| { + let start_block = block_addr_for_card(request.lba, raw.is_high_capacity())?; + let buffer = request + .segments + .first() + .copied() + .ok_or(BlkError::InvalidRequest)?; + if !buffer.len().is_multiple_of(BLOCK_SIZE) { + return Err(BlkError::Other("buffer is not block aligned")); + } + let ptr = NonNull::new(buffer.virt).ok_or(BlkError::Other("buffer pointer is null"))?; + let size = NonZeroUsize::new(buffer.len()).ok_or(BlkError::Other("buffer is empty"))?; + let dma = self.config.dma.as_ref(); + let id = match request.op { + rdif_block::RequestOp::Read => H::submit_read_request( + raw.host_mut(), + start_block, + ptr, + size, + dma, + &mut self.slot, + &mut self.pending, + )?, + rdif_block::RequestOp::Write => H::submit_write_request( + raw.host_mut(), + start_block, + ptr, + size, + dma, + &mut self.slot, + &mut self.pending, + )?, + rdif_block::RequestOp::Flush + | rdif_block::RequestOp::Discard + | rdif_block::RequestOp::WriteZeroes => return Err(BlkError::NotSupported), + }; + Ok(RequestId::new(usize::from(id))) + }) + } + + fn poll_request_inner(&mut self, request: RequestId) -> Result { + if let Some(index) = self.completed.iter().position(|id| *id == request) { + self.completed.swap_remove(index); + return Ok(RequestStatus::Complete); + } + self.poll_active_request(request) + } + + fn poll_active_request(&mut self, request: RequestId) -> Result { + let raw = self.raw.clone(); + match raw.with_mut(|raw| { + H::poll_block_request( + raw.host_mut(), + &mut self.pending, + BlockRequestId::new(usize::from(request)), + &mut self.slot, + ) + }) { + Ok(BlockPoll::Complete) => Ok(RequestStatus::Complete), + Ok(BlockPoll::Pending) => Ok(RequestStatus::Pending), + Err(err) => Err(map_dev_err_to_blk_err(err)), + } + } + + fn pending_id(&self) -> Option { + self.pending.as_ref().map(H::request_id) + } + + fn reap_pending_request(&mut self) -> Result { + let Some(active) = self.pending_id() else { + return Ok(RequestStatus::Complete); + }; + let id = RequestId::new(usize::from(active)); + match self.poll_active_request(id) { + Ok(RequestStatus::Complete) => { + self.completed.push(id); + Ok(RequestStatus::Complete) + } + Ok(RequestStatus::Pending) => Err(BlkError::Retry), + Err(err) => Err(err), + } + } +} + +// SAFETY: `BlockQueue` owns one pending request slot. The concrete host +// request object owns any borrowed request segment until task-side poll +// reports completion or error. +unsafe impl IQueue for BlockQueue +where + H: BlockHost, +{ + fn id(&self) -> usize { + self.id + } + + fn info(&self) -> rdif_block::QueueInfo { + rdif_block::QueueInfo { + id: self.id, + device: device_info(&self.config), + limits: queue_limits(&self.config, self.config.dma_mask), + } + } + + fn submit_request(&mut self, request: Request<'_>) -> Result { + self.submit_request_inner(request) + } + + fn poll_request(&mut self, request: RequestId) -> Result { + self.poll_request_inner(request) + } +} + +struct BlockIrqHandler +where + H: BlockHost, +{ + handle: ::IrqHandle, + _marker: PhantomData, +} + +impl rdif_block::IrqHandler for BlockIrqHandler +where + H: BlockHost, +{ + fn handle_irq(&self) -> rdif_block::Event { + let host_event = self.handle.handle_irq(); + let mut event = rdif_block::Event::none(); + if let Some(queue_id) = block_queue_ready_from_host_event(&host_event) { + event.push_queue(queue_id); + } + event + } +} + +pub fn queue_limits(config: &BlockConfig, dma_mask: u64) -> rdif_block::QueueLimits { + rdif_block::QueueLimits { + dma_mask, + dma_alignment: BLOCK_SIZE, + max_inflight: 1, + max_blocks_per_request: config.max_blocks_per_request, + max_segments: 1, + max_segment_size: config.max_segment_size, + supported_flags: rdif_block::RequestFlags::NONE, + supports_flush: false, + supports_discard: false, + supports_write_zeroes: false, + } +} + +pub fn device_info(config: &BlockConfig) -> rdif_block::DeviceInfo { + rdif_block::DeviceInfo { + name: Some(config.name), + ..rdif_block::DeviceInfo::new(config.capacity_blocks, BLOCK_SIZE) + } +} + +pub fn block_addr_for_card(block_id: u64, high_capacity: bool) -> Result { + let block_id = u32::try_from(block_id).map_err(|_| BlkError::InvalidBlockIndex(block_id))?; + if high_capacity { + Ok(block_id) + } else { + block_id + .checked_mul(BLOCK_SIZE as u32) + .ok_or(BlkError::InvalidBlockIndex(block_id as u64)) + } +} + +pub fn map_dev_err_to_blk_err(err: Error) -> BlkError { + match err { + Error::NoCard | Error::UnsupportedCommand | Error::CardLocked => BlkError::NotSupported, + Error::Misaligned | Error::InvalidArgument => { + BlkError::Other("SD/MMC request is not block aligned") + } + _ => BlkError::Io, + } +} + +pub fn transfer_mode_for_dma(dma: Option<&DeviceDma>) -> BlockTransferMode { + match dma { + Some(_) => BlockTransferMode::Dma, + None => BlockTransferMode::Fifo, + } +} + +pub fn can_fallback_to_fifo(err: Error) -> bool { + matches!( + err, + Error::UnsupportedCommand | Error::InvalidArgument | Error::Misaligned + ) +} + +struct SharedCore { + inner: Arc>, +} + +struct SharedCoreInner { + value: UnsafeCell, + borrowed: AtomicBool, +} + +struct SharedCoreGuard<'a, T> { + inner: &'a SharedCoreInner, +} + +// SAFETY: `SharedCore` serializes all mutable access through a single atomic +// borrow flag. IRQ top halves use host-specific cloneable handles instead. +unsafe impl Send for SharedCoreInner {} + +// SAFETY: See the `Send` impl. +unsafe impl Sync for SharedCoreInner {} + +impl SharedCore { + fn new(value: T) -> Self { + Self { + inner: Arc::new(SharedCoreInner { + value: UnsafeCell::new(value), + borrowed: AtomicBool::new(false), + }), + } + } + + fn with_mut(&self, f: impl FnOnce(&mut T) -> R) -> R { + let mut guard = self.inner.enter(); + f(guard.get_mut()) + } +} + +impl Clone for SharedCore { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + +impl SharedCoreInner { + fn enter(&self) -> SharedCoreGuard<'_, T> { + loop { + if let Some(guard) = self.try_enter() { + return guard; + } + core::hint::spin_loop(); + } + } + + fn try_enter(&self) -> Option> { + self.borrowed + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .ok()?; + Some(SharedCoreGuard { inner: self }) + } +} + +impl SharedCoreGuard<'_, T> { + fn get_mut(&mut self) -> &mut T { + unsafe { &mut *self.inner.value.get() } + } +} + +impl Drop for SharedCoreGuard<'_, T> { + fn drop(&mut self) { + self.inner.borrowed.store(false, Ordering::Release); + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use super::*; + use crate::{ + CommandResponsePoll, DataCommandPoll, + cmd::Command, + sdio::{ClockSpeed, HostEvent, HostEventKind}, + }; + + #[test] + fn fifo_config_limits_single_block_requests() { + let config = BlockConfig::fifo("test-sdmmc", 8, true); + let limits = queue_limits(&config, DEFAULT_DMA_MASK); + + assert_eq!(limits.max_inflight, 1); + assert_eq!(limits.max_blocks_per_request, 1); + assert_eq!(limits.max_segment_size, BLOCK_SIZE); + assert!(!limits.supports_flush); + } + + #[test] + fn disabled_irq_policy_does_not_advertise_sources() { + let device = BlockDevice::new( + SdioSdmmc::new(MockHost::default()), + BlockConfig::fifo("mock-sd", 8, false), + ); + + assert!(Interface::irq_sources(&device).is_empty()); + } + + #[test] + fn enabled_irq_handler_maps_host_event_to_queue_zero() { + let mut device = BlockDevice::new( + SdioSdmmc::new(MockHost::default()), + BlockConfig::fifo("mock-sd", 8, true), + ); + let handler = Interface::take_irq_handler(&mut device, 0).unwrap(); + + let event = handler.handle_irq(); + + assert!(event.queues.contains(0)); + assert!(!event.is_empty()); + } + + #[test] + fn poll_request_only_completes_matching_request_id() { + let mut queue = BlockQueue::::new( + SharedCore::new(SdioSdmmc::new(MockHost::default())), + BlockConfig::fifo("mock-sd", 8, false), + 0, + ); + queue.pending = Some(MockRequest { + id: BlockRequestId::new(7), + }); + + assert_eq!( + queue.poll_request_inner(RequestId::new(8)), + Ok(RequestStatus::Pending) + ); + assert_eq!( + queue.poll_request_inner(RequestId::new(7)), + Ok(RequestStatus::Complete) + ); + assert!(queue.pending.is_none()); + } + + #[test] + fn unsupported_ops_are_rejected() { + let mut queue = BlockQueue::::new( + SharedCore::new(SdioSdmmc::new(MockHost::default())), + BlockConfig::fifo("mock-sd", 8, false), + 0, + ); + let mut segments = []; + let request = Request { + op: rdif_block::RequestOp::Flush, + lba: 0, + block_count: 0, + segments: &mut segments, + flags: rdif_block::RequestFlags::NONE, + }; + + assert_eq!(queue.submit_request(request), Err(BlkError::NotSupported)); + } + + #[derive(Clone, Default)] + struct MockIrqHandle; + + impl SdioIrqHandle for MockIrqHandle { + type Event = MockEvent; + + fn handle_irq(&self) -> Self::Event { + MockEvent(HostEventKind::TransferComplete) + } + } + + #[derive(Clone, Copy, Default)] + struct MockEvent(HostEventKind); + + impl HostEvent for MockEvent { + fn kind(&self) -> HostEventKind { + self.0 + } + } + + #[derive(Default)] + struct MockHost { + irq_enabled: AtomicBool, + next_id: AtomicUsize, + } + + #[derive(Default)] + struct MockSlot; + + struct MockRequest { + id: BlockRequestId, + } + + impl SdioHost for MockHost { + type Event = MockEvent; + type DataRequest<'a> = (); + + fn submit_command(&mut self, _cmd: &Command) -> Result<(), Error> { + Err(Error::UnsupportedCommand) + } + + fn poll_command_response(&mut self) -> Result { + Ok(CommandResponsePoll::Pending) + } + + fn submit_read_data<'a>( + &mut self, + _cmd: &Command, + _buf: &'a mut [u8], + _block_size: u32, + _block_count: u32, + ) -> Result, Error> { + Err(Error::UnsupportedCommand) + } + + fn submit_write_data<'a>( + &mut self, + _cmd: &Command, + _buf: &'a [u8], + _block_size: u32, + _block_count: u32, + ) -> Result, Error> { + Err(Error::UnsupportedCommand) + } + + fn poll_data_request<'a>( + &mut self, + _request: &mut Self::DataRequest<'a>, + ) -> Result { + Err(Error::UnsupportedCommand) + } + + fn set_bus_width(&mut self, _width: crate::sdio::BusWidth) -> Result<(), Error> { + Ok(()) + } + + fn set_clock(&mut self, _speed: ClockSpeed) -> Result<(), Error> { + Ok(()) + } + + fn enable_completion_irq(&mut self) -> Result<(), Error> { + self.irq_enabled.store(true, Ordering::Release); + Ok(()) + } + + fn disable_completion_irq(&mut self) -> Result<(), Error> { + self.irq_enabled.store(false, Ordering::Release); + Ok(()) + } + } + + impl SdioIrqHost for MockHost { + type IrqHandle = MockIrqHandle; + + fn irq_handle(&self) -> Self::IrqHandle { + MockIrqHandle + } + + fn completion_irq_enabled(&self) -> bool { + self.irq_enabled.load(Ordering::Acquire) + } + } + + impl BlockHost for MockHost { + type Request = MockRequest; + type Slot = MockSlot; + + fn submit_read_request( + &mut self, + _start_block: u32, + _buffer: NonNull, + _size: NonZeroUsize, + _dma: Option<&DeviceDma>, + _slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + self.submit_mock_request(pending) + } + + fn submit_write_request( + &mut self, + _start_block: u32, + _buffer: NonNull, + _size: NonZeroUsize, + _dma: Option<&DeviceDma>, + _slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + self.submit_mock_request(pending) + } + + fn poll_block_request( + &mut self, + pending: &mut Option, + request: BlockRequestId, + _slot: &mut Self::Slot, + ) -> Result { + match pending.as_ref() { + Some(active) if active.id == request => { + *pending = None; + Ok(BlockPoll::Complete) + } + Some(_) => Ok(BlockPoll::Pending), + None => Ok(BlockPoll::Complete), + } + } + + fn request_id(request: &Self::Request) -> BlockRequestId { + request.id + } + } + + impl MockHost { + fn submit_mock_request( + &self, + pending: &mut Option, + ) -> Result { + if pending.is_some() { + return Err(BlkError::Retry); + } + let id = BlockRequestId::new(self.next_id.fetch_add(1, Ordering::Relaxed)); + *pending = Some(MockRequest { id }); + Ok(id) + } + } +} diff --git a/drivers/blk/sdmmc-protocol/src/sdio.rs b/drivers/blk/sdmmc-protocol/src/sdio.rs index d3acd60132..a7723346d1 100644 --- a/drivers/blk/sdmmc-protocol/src/sdio.rs +++ b/drivers/blk/sdmmc-protocol/src/sdio.rs @@ -1530,6 +1530,11 @@ impl SdioSdmmc { request.mmc_switch_request = None; if let Err(_e) = self.host.set_clock(ClockSpeed::HighSpeed) { debug!("sdio: host refused HighSpeed clock ({:?})", _e); + } else { + info!( + "sdio: MMC speed selected HighSpeed bus_width={:?}", + self.bus_width + ); } request.state = SdioInitState::Complete; Ok(OperationPoll::Pending) @@ -1689,9 +1694,19 @@ impl SdioSdmmc { SdioInitState::Complete => { let kind = request.kind.ok_or(Error::InvalidArgument)?; let ocr = request.ocr.ok_or(Error::InvalidArgument)?; + let ext_csd_timing = request.parsed_ext_csd.as_ref().map(|csd| csd.timing()); + let ext_csd_bus_width = request.parsed_ext_csd.as_ref().map(|csd| csd.bus_width()); info!( - "sdio: init done kind={:?} sd_v2={} high_capacity={} rca={:#x} ocr={:#x}", - kind, request.sd_v2, self.high_capacity, self.rca, ocr.raw + "sdio: init done kind={:?} sd_v2={} high_capacity={} rca={:#x} ocr={:#x} \ + host_bus_width={:?} ext_csd_bus_width={:?} ext_csd_timing={:?}", + kind, + request.sd_v2, + self.high_capacity, + self.rca, + ocr.raw, + self.bus_width, + ext_csd_bus_width, + ext_csd_timing ); Ok(OperationPoll::Complete(CardInfo { kind,