diff --git a/Cargo.lock b/Cargo.lock index 00dad97399..8c99871d1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3052,6 +3052,8 @@ dependencies = [ "embedded-hal", "log", "mmio-api", + "rdif-block", + "sdio-host2", "sdmmc-protocol", "volatile 0.6.1", ] @@ -5729,6 +5731,8 @@ dependencies = [ "dma-api", "log", "mmio-api", + "rdif-block", + "sdio-host2", "sdmmc-protocol", "volatile 0.6.1", ] @@ -6353,6 +6357,7 @@ version = "0.9.1" dependencies = [ "dma-api", "rdif-base", + "spin 0.12.0", "thiserror 2.0.18", ] @@ -7146,6 +7151,8 @@ dependencies = [ "embedded-hal", "log", "mmio-api", + "rdif-block", + "sdio-host2", "sdmmc-protocol", ] @@ -7153,13 +7160,23 @@ dependencies = [ name = "sdio-host-cv1800" version = "0.1.0" +[[package]] +name = "sdio-host2" +version = "0.1.0" +dependencies = [ + "dma-api", +] + [[package]] name = "sdmmc-protocol" version = "0.1.3" dependencies = [ "bitflags 2.13.0", + "dma-api", "embedded-hal", "log", + "rdif-block", + "sdio-host2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b3c7da8b08..7bdf9d9307 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ members = [ "components/scope-local", "components/sdhci-cv1800", "components/sdio-host", + "components/sdio-host2", "components/someboot", "components/somehal-macros", "components/starry-process", @@ -242,6 +243,7 @@ rdrive-macros = { version = "0.4.2", path = "drivers/rdrive-macros" } rsext4 = { version = "0.7.2", path = "components/rsext4" } scope-local = { version = "0.4.0", path = "components/scope-local" } sdio-host = { version = "0.1.0", package = "sdio-host-cv1800", path = "components/sdio-host" } +sdio-host2 = { version = "0.1.0", path = "components/sdio-host2" } sdhci-cv1800 = { version = "0.1.1", path = "components/sdhci-cv1800" } aic8800 = { version = "0.2.0", path = "components/aic8800" } crab-usb = { version = "0.9.8", path = "drivers/usb/usb-host" } diff --git a/apps/starry/block-rw-bench/README.md b/apps/starry/block-rw-bench/README.md new file mode 100644 index 0000000000..542468bf60 --- /dev/null +++ b/apps/starry/block-rw-bench/README.md @@ -0,0 +1,11 @@ +# block-rw-bench + +Starry board file-I/O benchmark for SD/MMC RDIF validation. + +Build and install the helper into the board rootfs as `/usr/bin/block-rw-bench` +before running `cargo xtask starry app board -t block-rw-bench -b `. +The board app `init.sh` executes that helper directly. + +The helper writes files under `/root/block-rw-bench/`, calls `sync_all`, reads +the data back, verifies a deterministic pattern, and prints one result line for +each block size plus a final success line. diff --git a/apps/starry/block-rw-bench/block-rw-bench/.gitignore b/apps/starry/block-rw-bench/block-rw-bench/.gitignore new file mode 100644 index 0000000000..ea8c4bf7f3 --- /dev/null +++ b/apps/starry/block-rw-bench/block-rw-bench/.gitignore @@ -0,0 +1 @@ +/target diff --git a/apps/starry/block-rw-bench/block-rw-bench/Cargo.lock b/apps/starry/block-rw-bench/block-rw-bench/Cargo.lock new file mode 100644 index 0000000000..1bf342cee2 --- /dev/null +++ b/apps/starry/block-rw-bench/block-rw-bench/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-rw-bench" +version = "0.1.0" diff --git a/apps/starry/block-rw-bench/block-rw-bench/Cargo.toml b/apps/starry/block-rw-bench/block-rw-bench/Cargo.toml new file mode 100644 index 0000000000..8263527784 --- /dev/null +++ b/apps/starry/block-rw-bench/block-rw-bench/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "block-rw-bench" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" + +[dependencies] + +[workspace] diff --git a/apps/starry/block-rw-bench/block-rw-bench/src/main.rs b/apps/starry/block-rw-bench/block-rw-bench/src/main.rs new file mode 100644 index 0000000000..18033e5f19 --- /dev/null +++ b/apps/starry/block-rw-bench/block-rw-bench/src/main.rs @@ -0,0 +1,136 @@ +use std::{ + env, + fs::{self, File, OpenOptions}, + io::{self, BufReader, Read, Write}, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; + +const BENCH_DIR: &str = "/root/block-rw-bench"; +const TOTAL_BYTES: usize = 64 * 1024 * 1024; +const BLOCK_SIZES: [usize; 3] = [4 * 1024, 64 * 1024, 1024 * 1024]; +const DROP_CACHES_ENV: &str = "BLOCK_RW_BENCH_DROP_CACHES"; + +fn main() { + if let Err(err) = run() { + eprintln!("block-rw-bench: error: {err}"); + std::process::exit(1); + } +} + +fn run() -> io::Result<()> { + let dir = Path::new(BENCH_DIR); + fs::create_dir_all(dir)?; + + for &block_size in &BLOCK_SIZES { + run_case(dir, block_size, TOTAL_BYTES)?; + } + + println!( + "block-rw-bench: done cases={} status=ok", + BLOCK_SIZES.len() + ); + Ok(()) +} + +fn run_case(dir: &Path, block_size: usize, bytes: usize) -> io::Result<()> { + maybe_drop_caches()?; + + let path = case_path(dir, block_size); + let mut pattern = vec![0; block_size]; + let write_start = Instant::now(); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&path)?; + + let mut offset = 0usize; + while offset < bytes { + let chunk_len = (bytes - offset).min(block_size); + fill_pattern(&mut pattern[..chunk_len], block_size, offset); + file.write_all(&pattern[..chunk_len])?; + offset += chunk_len; + } + let write_elapsed = write_start.elapsed(); + + let fsync_start = Instant::now(); + file.sync_all()?; + let fsync_elapsed = fsync_start.elapsed(); + drop(file); + + maybe_drop_caches()?; + + let read_start = Instant::now(); + verify_file(&path, block_size, bytes)?; + let read_elapsed = read_start.elapsed(); + + println!( + "block-rw-bench: case block_size={} bytes={} write_mib_s={:.2} read_mib_s={:.2} fsync_ms={} verify=ok", + block_size, + bytes, + throughput_mib_s(bytes, write_elapsed), + throughput_mib_s(bytes, read_elapsed), + duration_ms(fsync_elapsed) + ); + + fs::remove_file(path)?; + Ok(()) +} + +fn verify_file(path: &Path, block_size: usize, bytes: usize) -> io::Result<()> { + let mut reader = BufReader::new(File::open(path)?); + let mut actual = vec![0; block_size]; + let mut expected = vec![0; block_size]; + let mut offset = 0usize; + + while offset < bytes { + let chunk_len = (bytes - offset).min(block_size); + reader.read_exact(&mut actual[..chunk_len])?; + fill_pattern(&mut expected[..chunk_len], block_size, offset); + if actual[..chunk_len] != expected[..chunk_len] { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "verify mismatch block_size={} offset={} expected={:02x} actual={:02x}", + block_size, offset, expected[0], actual[0] + ), + )); + } + offset += chunk_len; + } + + Ok(()) +} + +fn fill_pattern(buf: &mut [u8], block_size: usize, base_offset: usize) { + let seed = block_size as u64 ^ 0x5d51_d1f5_a5a5_1234; + for (index, byte) in buf.iter_mut().enumerate() { + let pos = (base_offset + index) as u64; + *byte = pos + .wrapping_mul(1103515245) + .wrapping_add(seed) + .rotate_left((pos & 7) as u32) as u8; + } +} + +fn throughput_mib_s(bytes: usize, elapsed: Duration) -> f64 { + let seconds = elapsed.as_secs_f64().max(0.000_001); + bytes as f64 / (1024.0 * 1024.0) / seconds +} + +fn duration_ms(elapsed: Duration) -> u128 { + elapsed.as_millis() +} + +fn case_path(dir: &Path, block_size: usize) -> PathBuf { + dir.join(format!("case-{}.bin", block_size)) +} + +fn maybe_drop_caches() -> io::Result<()> { + if env::var_os(DROP_CACHES_ENV).is_none() { + return Ok(()); + } + + fs::write("/proc/sys/vm/drop_caches", b"3\n") +} diff --git a/apps/starry/block-rw-bench/board-orangepi-5-plus.toml b/apps/starry/block-rw-bench/board-orangepi-5-plus.toml new file mode 100644 index 0000000000..7554d43b03 --- /dev/null +++ b/apps/starry/block-rw-bench/board-orangepi-5-plus.toml @@ -0,0 +1,13 @@ +board_type = "OrangePi-5-Plus" +shell_prefix = "root@starry:/root #" +success_regex = [ + '(?m)^block-rw-bench: done cases=3 status=ok$', +] +fail_regex = [ + '(?i)\bpanic(?:ked)?\b', + '(?m)^block-rw-bench: error: .*$', + '(?m)^block-rw-bench: .*verify=(?:fail|mismatch).*$', + '(?i)(block-rw-bench|sh): .*not found', + '(?i)\bI/O error\b', +] +timeout = 600 diff --git a/apps/starry/block-rw-bench/board-phytiumpi.toml b/apps/starry/block-rw-bench/board-phytiumpi.toml new file mode 100644 index 0000000000..52c2b3ae08 --- /dev/null +++ b/apps/starry/block-rw-bench/board-phytiumpi.toml @@ -0,0 +1,13 @@ +board_type = "PhytiumPi" +shell_prefix = "root@starry:/root #" +success_regex = [ + '(?m)^block-rw-bench: done cases=3 status=ok$', +] +fail_regex = [ + '(?i)\bpanic(?:ked)?\b', + '(?m)^block-rw-bench: error: .*$', + '(?m)^block-rw-bench: .*verify=(?:fail|mismatch).*$', + '(?i)(block-rw-bench|sh): .*not found', + '(?i)\bI/O error\b', +] +timeout = 600 diff --git a/apps/starry/block-rw-bench/board-roc-rk3568-pc.toml b/apps/starry/block-rw-bench/board-roc-rk3568-pc.toml new file mode 100644 index 0000000000..0043a5fa16 --- /dev/null +++ b/apps/starry/block-rw-bench/board-roc-rk3568-pc.toml @@ -0,0 +1,13 @@ +board_type = "ROC-RK3568-PC" +shell_prefix = "root@starry:/root #" +success_regex = [ + '(?m)^block-rw-bench: done cases=3 status=ok$', +] +fail_regex = [ + '(?i)\bpanic(?:ked)?\b', + '(?m)^block-rw-bench: error: .*$', + '(?m)^block-rw-bench: .*verify=(?:fail|mismatch).*$', + '(?i)(block-rw-bench|sh): .*not found', + '(?i)\bI/O error\b', +] +timeout = 1200 diff --git a/apps/starry/block-rw-bench/build-aarch64-unknown-none-softfloat.toml b/apps/starry/block-rw-bench/build-aarch64-unknown-none-softfloat.toml new file mode 100644 index 0000000000..53e7e53daa --- /dev/null +++ b/apps/starry/block-rw-bench/build-aarch64-unknown-none-softfloat.toml @@ -0,0 +1,9 @@ +target = "aarch64-unknown-none-softfloat" +features = [ + "ax-driver/rockchip-soc", + "ax-driver/rockchip-sdhci", + "ax-driver/rockchip-dwmmc", + "ax-driver/phytium-mci", +] +log = "Info" +max_cpu_num = 1 diff --git a/apps/starry/block-rw-bench/init.sh b/apps/starry/block-rw-bench/init.sh new file mode 100644 index 0000000000..73f919c280 --- /dev/null +++ b/apps/starry/block-rw-bench/init.sh @@ -0,0 +1 @@ +rm -rf /root/block-rw-bench && mkdir -p /root/block-rw-bench && /usr/bin/block-rw-bench && sync diff --git a/components/axklib/src/dma.rs b/components/axklib/src/dma.rs index 21e7ff0510..d3f9ad766f 100644 --- a/components/axklib/src/dma.rs +++ b/components/axklib/src/dma.rs @@ -2,7 +2,8 @@ use core::{alloc::Layout, num::NonZeroUsize, ptr::NonNull}; use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr}; use dma_api::{ - DeviceDma, DmaAllocHandle, DmaConstraints, DmaDirection, DmaError, DmaMapHandle, DmaOp, + DeviceDma, DmaAllocHandle, DmaConstraints, DmaDirection, DmaDomainId, DmaError, DmaMapHandle, + DmaOp, }; pub struct KlibDma; @@ -13,8 +14,12 @@ pub fn op() -> &'static KlibDma { &DMA } +pub const fn domain_id() -> DmaDomainId { + DmaDomainId::legacy_global() +} + pub fn device_with_mask(dma_mask: u64) -> DeviceDma { - DeviceDma::new(dma_mask, op()) + DeviceDma::new(domain_id(), dma_mask, op()) } struct DmaPages { diff --git a/components/sdio-host2/Cargo.toml b/components/sdio-host2/Cargo.toml new file mode 100644 index 0000000000..4e29d9487b --- /dev/null +++ b/components/sdio-host2/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "sdio-host2" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +description = "no_std SD/SDIO/MMC host bus transaction traits and types" +keywords.workspace = true +categories.workspace = true + +[dependencies] +dma-api.workspace = true diff --git a/components/sdio-host2/src/lib.rs b/components/sdio-host2/src/lib.rs new file mode 100644 index 0000000000..19b0e80cef --- /dev/null +++ b/components/sdio-host2/src/lib.rs @@ -0,0 +1,785 @@ +//! Physical SD/SDIO/MMC host-bus transaction traits. +//! +//! This crate intentionally models the shared CMD/DAT bus rather than a card, +//! block device, filesystem, or runtime queue. A host accepts one transaction +//! at a time: a command, an optional data phase, and a task-side poll path to +//! observe completion. Higher-level SD/MMC card protocols live in +//! `sdmmc-protocol`. + +#![no_std] + +extern crate alloc; + +use alloc::boxed::Box; +use core::{ + fmt, + num::{NonZeroU16, NonZeroU32}, +}; + +use dma_api::{CompletedDma, DmaDirection, PreparedDma}; + +/// SD/SDIO/MMC command packet submitted on the CMD line. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Command { + pub index: u8, + pub argument: u32, + pub response: ResponseType, +} + +impl Command { + pub const fn new(index: u8, argument: u32, response: ResponseType) -> Self { + Self { + index, + argument, + response, + } + } + + pub const fn index(self) -> u8 { + self.index + } + + pub const fn argument(self) -> u32 { + self.argument + } + + pub const fn with_response(self, response: ResponseType) -> Self { + Self { response, ..self } + } + + /// Return a copy of this command with its response type overridden. + /// + /// Kept as a compatibility alias for existing SD/MMC protocol helpers. + pub const fn with_resp_type(self, response: ResponseType) -> Self { + self.with_response(response) + } + + /// Compatibility alias for older SD/MMC command helpers. + pub const fn cmd(self) -> u8 { + self.index + } + + /// Compatibility alias for older SD/MMC command helpers. + pub const fn arg(self) -> u32 { + self.argument + } + + /// Direction of the data phase that follows this command when it is + /// unambiguous from the command index alone. + /// + /// SDIO CMD53 carries its direction in the argument; CMD6 is also + /// overloaded between ACMD6 and SWITCH_FUNC, so both return `None`. + pub const fn data_direction(&self) -> Option { + match self.index { + 17 | 18 => Some(DataDirection::Read), + 24 | 25 => Some(DataDirection::Write), + _ => None, + } + } + + /// Size in bytes of the data block when fixed by the command index. + pub const fn data_block_size(&self) -> Option { + match self.index { + 17 | 18 | 24 | 25 => Some(512), + _ => None, + } + } + + /// Compute the SD SPI-mode CRC7 for this command packet. + pub fn crc7(&self) -> u8 { + let mut crc: u8 = 0; + let token: u8 = 0x40 | (self.index & 0x3F); + crc = crc7_update(crc, token); + for byte in self.argument.to_be_bytes() { + crc = crc7_update(crc, byte); + } + (crc << 1) | 1 + } + + /// Build the 6-byte SD SPI command packet. + pub fn to_spi_bytes(&self) -> [u8; 6] { + let crc = self.crc7(); + let token = 0x40 | (self.index & 0x3F); + let arg = self.argument.to_be_bytes(); + [token, arg[0], arg[1], arg[2], arg[3], crc] + } +} + +fn crc7_update(crc: u8, byte: u8) -> u8 { + let mut crc = crc; + let mut data = byte; + for _ in 0..8 { + crc <<= 1; + if (crc ^ data) & 0x80 != 0 { + crc ^= 0x89; + } + data <<= 1; + } + crc +} + +/// Command response shape expected from the card. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ResponseType { + None, + R1, + R1b, + R2, + R3, + R4, + R5, + R6, + R7, +} + +/// Raw response words harvested by a host controller. +/// +/// ABI: +/// +/// - 48-bit responses store their response payload in `words[0]`. +/// - R2/CID/CSD responses store four 32-bit words in most-significant-word +/// first order. +/// - Each word is the big-endian value of the corresponding response bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RawResponse { + pub ty: ResponseType, + pub words: [u32; 4], +} + +impl RawResponse { + pub const fn new(ty: ResponseType, words: [u32; 4]) -> Self { + Self { ty, words } + } + + pub const fn empty() -> Self { + Self { + ty: ResponseType::None, + words: [0; 4], + } + } +} + +/// Direction of a data phase on DAT lines. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum DataDirection { + Read, + Write, +} + +/// Caller-owned data buffer tied to an in-flight transaction lifetime. +pub enum DataBuffer<'a> { + Read(&'a mut [u8]), + Write(&'a [u8]), + Dma(PreparedDma), +} + +impl DataBuffer<'_> { + pub fn len(&self) -> usize { + match self { + Self::Read(buf) => buf.len(), + Self::Write(buf) => buf.len(), + Self::Dma(buffer) => buffer.len().get(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn matches_direction(&self, direction: DataDirection) -> bool { + match self { + Self::Read(_) => direction == DataDirection::Read, + Self::Write(_) => direction == DataDirection::Write, + Self::Dma(buffer) => matches!( + (buffer.direction(), direction), + (DmaDirection::FromDevice, DataDirection::Read) + | (DmaDirection::ToDevice, DataDirection::Write) + | (DmaDirection::Bidirectional, _) + ), + } + } +} + +pub type DataTransfer<'a> = DataBuffer<'a>; + +/// Error returned while constructing an owned-DMA data phase. +pub struct DmaPhaseError { + error: Error, + buffer: Box, +} + +impl DmaPhaseError { + fn new(error: Error, buffer: PreparedDma) -> Self { + Self { + error, + buffer: Box::new(buffer), + } + } + + pub const fn error(&self) -> Error { + self.error + } + + pub fn into_buffer(self) -> PreparedDma { + *self.buffer + } + + pub fn into_parts(self) -> (Error, PreparedDma) { + (self.error, *self.buffer) + } +} + +impl fmt::Debug for DmaPhaseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DmaPhaseError") + .field("error", &self.error) + .finish_non_exhaustive() + } +} + +impl fmt::Display for DmaPhaseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.error.fmt(f) + } +} + +impl core::error::Error for DmaPhaseError {} + +/// Optional data phase associated with a command. +pub struct DataPhase<'a> { + pub direction: DataDirection, + pub block_size: NonZeroU16, + pub block_count: NonZeroU32, + pub buffer: DataBuffer<'a>, +} + +impl<'a> DataPhase<'a> { + pub fn read( + block_size: NonZeroU16, + block_count: NonZeroU32, + buffer: &'a mut [u8], + ) -> Result { + let phase = Self { + direction: DataDirection::Read, + block_size, + block_count, + buffer: DataBuffer::Read(buffer), + }; + phase.validate()?; + Ok(phase) + } + + pub fn write( + block_size: NonZeroU16, + block_count: NonZeroU32, + buffer: &'a [u8], + ) -> Result { + let phase = Self { + direction: DataDirection::Write, + block_size, + block_count, + buffer: DataBuffer::Write(buffer), + }; + phase.validate()?; + Ok(phase) + } + + pub fn dma( + direction: DataDirection, + block_size: NonZeroU16, + block_count: NonZeroU32, + buffer: PreparedDma, + ) -> Result { + let phase = Self { + direction, + block_size, + block_count, + buffer: DataBuffer::Dma(buffer), + }; + match phase.validate() { + Ok(()) => Ok(phase), + Err(err) => { + let DataBuffer::Dma(buffer) = phase.buffer else { + unreachable!("DataPhase::dma always stores a DMA buffer") + }; + Err(DmaPhaseError::new(err, buffer)) + } + } + } + + pub fn validate(&self) -> Result<(), Error> { + let expected = usize::from(self.block_size.get()) + .checked_mul( + usize::try_from(self.block_count.get()).map_err(|_| Error::InvalidArgument)?, + ) + .ok_or(Error::InvalidArgument)?; + if self.buffer.len() != expected { + return Err(Error::InvalidArgument); + } + if !self.buffer.matches_direction(self.direction) { + return Err(Error::InvalidArgument); + } + Ok(()) + } +} + +/// One physical bus transaction: a command and an optional data phase. +pub struct Transaction<'a> { + pub command: Command, + pub data: Option>, +} + +impl<'a> Transaction<'a> { + pub const fn command(command: Command) -> Self { + Self { + command, + data: None, + } + } + + pub const fn with_data(command: Command, data: DataPhase<'a>) -> Self { + Self { + command, + data: Some(data), + } + } +} + +/// Submit failure for an owned transaction. +/// +/// When `transaction` is present, the caller may recover and retry the DMA +/// backing. When it is absent, the host had to consume/quiesce the transaction +/// while handling the error; no hardware access remains active on return. +pub struct SubmitTransactionError<'a> { + pub error: Error, + transaction: Option>>, +} + +impl<'a> SubmitTransactionError<'a> { + pub fn new(error: Error, transaction: Transaction<'a>) -> Self { + Self { + error, + transaction: Some(Box::new(transaction)), + } + } + + pub const fn consumed(error: Error) -> Self { + Self { + error, + transaction: None, + } + } + + pub fn into_transaction(self) -> Option> { + self.transaction.map(|transaction| *transaction) + } +} + +/// Result of advancing a submitted request once. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RequestPoll { + Pending, + Ready(Result), +} + +/// Error returned when a request is polled through the wrong handle or after +/// its terminal state. +/// +/// Unlike [`RequestPoll::Ready`], this is not a transfer terminal state for +/// the request payload. Implementations must not report a terminal +/// [`RequestPoll::Ready`] error until the controller is no longer accessing +/// the transaction buffer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum PollRequestError { + WrongOwner, + WrongKind, + AlreadyCompleted, + StaleGeneration, + /// Recovery could not be reported through the requested handle. + /// + /// Safe host implementations must still quiesce the hardware before any + /// request object that borrows caller memory can be dropped. This variant + /// is diagnostic only; it must not mean DMA is still active. + RecoveryFailed, +} + +/// SD/SDIO/MMC bus width. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum BusWidth { + Bit1, + Bit4, + Bit8, +} + +/// Named card clock modes used by SD/MMC protocol state machines. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ClockSpeed { + Identification, + Default, + HighSpeed, + Sdr12, + Sdr25, + Sdr50, + Sdr104, + Ddr50, + Hs200, +} + +/// Concrete clock frequency request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct ClockHz(pub u32); + +/// Bus signaling voltage. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum SignalVoltage { + V330, + V180, + V120, +} + +/// Non-data bus operation that may itself need asynchronous completion. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum BusOp { + ResetAll, + ResetCommandLine, + ResetDataLine, + PowerOn, + PowerOff, + SetClock(ClockSpeed), + SetClockHz(ClockHz), + SetBusWidth(BusWidth), + SetSignalVoltage(SignalVoltage), + ExecuteTuning { + command: Command, + block_size: NonZeroU16, + }, +} + +/// Host/bus-layer error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + Busy, + Timeout, + Crc, + NoCard, + Unsupported, + InvalidArgument, + Misaligned, + Bus, + Controller, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match self { + Self::Busy => "host bus is busy", + Self::Timeout => "host bus timeout", + Self::Crc => "host bus CRC error", + Self::NoCard => "no card present", + Self::Unsupported => "operation is not supported", + Self::InvalidArgument => "invalid host bus argument", + Self::Misaligned => "misaligned host bus buffer", + Self::Bus => "host bus error", + Self::Controller => "host controller error", + }; + f.write_str(s) + } +} + +impl core::error::Error for Error {} + +impl fmt::Display for PollRequestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match self { + Self::WrongOwner => "request belongs to a different host", + Self::WrongKind => "request was polled through the wrong operation kind", + Self::AlreadyCompleted => "request has already completed", + Self::StaleGeneration => "request generation is no longer active", + Self::RecoveryFailed => "request recovery failed", + }; + f.write_str(s) + } +} + +impl core::error::Error for PollRequestError {} + +/// Physical SD/SDIO/MMC host bus. +/// +/// The base contract is single active transaction: a host may reject a submit +/// with [`Error::Busy`] while another transaction or bus operation is active. +pub trait SdioHost { + type TransactionRequest<'a> + where + Self: 'a; + type BusRequest; + + /// Submit one CMD/DAT transaction. + /// + /// # Safety + /// + /// Callers must poll the returned request until [`RequestPoll::Ready`] or + /// call [`Self::abort_transaction`] before dropping it. Until one of those + /// terminal paths runs, the host may still access the associated data + /// buffer through DMA or FIFO PIO. + unsafe fn submit_transaction<'a>( + &mut self, + transaction: Transaction<'a>, + ) -> Result, Error> + where + Self: 'a; + + /// Submit one CMD/DAT transaction while preserving transaction ownership + /// on submit-side failure when the host has not started hardware access. + /// + /// The default path is kept for legacy hosts. Native DMA users should + /// override it so submit failure can return the original transaction. + /// + /// # Safety + /// + /// Same lifetime contract as [`Self::submit_transaction`]. + unsafe fn submit_transaction_owned<'a>( + &mut self, + transaction: Transaction<'a>, + ) -> Result, SubmitTransactionError<'a>> + where + Self: 'a, + { + match unsafe { self.submit_transaction(transaction) } { + Ok(request) => Ok(request), + Err(error) => Err(SubmitTransactionError::consumed(error)), + } + } + + fn poll_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result, PollRequestError> + where + Self: 'a; + + /// Abort a transaction. + /// + /// This is part of the safe lifetime contract for borrowed transaction + /// buffers. Implementations may return an error to report that the + /// controller had to be reset or poisoned, but before returning they must + /// have stopped command/data engines and any DMA bus-master access that + /// could still touch the request buffer. + fn abort_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result<(), Error> + where + Self: 'a; + + fn take_completed_dma<'a>( + &mut self, + _request: &mut Self::TransactionRequest<'a>, + ) -> Option + where + Self: 'a, + { + None + } + + /// Submit one non-data bus operation. + /// + /// # Safety + /// + /// The returned request must be polled until [`RequestPoll::Ready`] or + /// passed to [`Self::abort_bus_op`] before being dropped. + unsafe fn submit_bus_op(&mut self, op: BusOp) -> Result; + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, PollRequestError>; + + /// Abort a bus operation. + /// + /// Like [`Self::abort_transaction`], returning from this method means the + /// controller is no longer executing the operation even when the return + /// value carries a diagnostic error. + fn abort_bus_op(&mut self, request: &mut Self::BusRequest) -> Result<(), Error>; + + fn now_ms(&self) -> Option { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct MockHost { + busy: bool, + } + + #[derive(Debug)] + struct MockTransactionRequest { + response: RawResponse, + pending_once: bool, + done: bool, + } + + #[derive(Debug)] + struct MockBusRequest { + pending_once: bool, + done: bool, + } + + impl SdioHost for MockHost { + type TransactionRequest<'a> + = MockTransactionRequest + where + Self: 'a; + type BusRequest = MockBusRequest; + + unsafe fn submit_transaction<'a>( + &mut self, + transaction: Transaction<'a>, + ) -> Result, Error> + where + Self: 'a, + { + if self.busy { + return Err(Error::Busy); + } + self.busy = true; + Ok(MockTransactionRequest { + response: RawResponse::new(transaction.command.response, [0x1234, 0, 0, 0]), + pending_once: true, + done: false, + }) + } + + fn poll_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result, PollRequestError> + where + Self: 'a, + { + if request.done { + return Err(PollRequestError::AlreadyCompleted); + } + if request.pending_once { + request.pending_once = false; + return Ok(RequestPoll::Pending); + } + self.busy = false; + request.done = true; + Ok(RequestPoll::Ready(Ok(request.response))) + } + + fn abort_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result<(), Error> + where + Self: 'a, + { + request.done = true; + self.busy = false; + Ok(()) + } + + unsafe fn submit_bus_op(&mut self, _op: BusOp) -> Result { + if self.busy { + return Err(Error::Busy); + } + self.busy = true; + Ok(MockBusRequest { + pending_once: false, + done: false, + }) + } + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, PollRequestError> { + if request.done { + return Err(PollRequestError::AlreadyCompleted); + } + if request.pending_once { + request.pending_once = false; + return Ok(RequestPoll::Pending); + } + self.busy = false; + request.done = true; + Ok(RequestPoll::Ready(Ok(()))) + } + + fn abort_bus_op(&mut self, request: &mut Self::BusRequest) -> Result<(), Error> { + request.done = true; + self.busy = false; + Ok(()) + } + } + + #[test] + fn data_phase_validates_buffer_shape() { + let mut read = [0u8; 1024]; + let block = NonZeroU16::new(512).unwrap(); + let phase = DataPhase::read(block, NonZeroU32::new(2).unwrap(), &mut read).unwrap(); + assert_eq!(phase.direction, DataDirection::Read); + assert_eq!(phase.buffer.len(), 1024); + } + + #[test] + fn host_reports_busy_for_second_active_transaction() { + let mut host = MockHost { busy: false }; + let cmd = Command::new(17, 0, ResponseType::R1); + let mut request = unsafe { host.submit_transaction(Transaction::command(cmd)) }.unwrap(); + assert_eq!( + unsafe { host.submit_transaction(Transaction::command(cmd)) }.unwrap_err(), + Error::Busy + ); + assert_eq!( + host.poll_transaction(&mut request), + Ok(RequestPoll::Pending) + ); + assert!(matches!( + host.poll_transaction(&mut request), + Ok(RequestPoll::Ready(Ok(_))) + )); + assert_eq!( + host.poll_transaction(&mut request), + Err(PollRequestError::AlreadyCompleted) + ); + assert!(unsafe { host.submit_transaction(Transaction::command(cmd)) }.is_ok()); + } + + #[test] + fn bus_op_uses_same_single_active_contract() { + let mut host = MockHost { busy: false }; + let _request = unsafe { host.submit_bus_op(BusOp::SetClock(ClockSpeed::Default)) }.unwrap(); + assert_eq!( + unsafe { host.submit_bus_op(BusOp::SetBusWidth(BusWidth::Bit4)) }.unwrap_err(), + Error::Busy + ); + } + + #[test] + fn abort_releases_single_active_contract() { + let mut host = MockHost { busy: false }; + let cmd = Command::new(17, 0, ResponseType::R1); + let mut request = unsafe { host.submit_transaction(Transaction::command(cmd)) }.unwrap(); + + host.abort_transaction(&mut request).unwrap(); + + assert!(unsafe { host.submit_transaction(Transaction::command(cmd)) }.is_ok()); + assert_eq!( + host.poll_transaction(&mut request), + Err(PollRequestError::AlreadyCompleted) + ); + } +} diff --git a/drivers/ax-driver/Cargo.toml b/drivers/ax-driver/Cargo.toml index 5d879972d7..cf0854b5f0 100644 --- a/drivers/ax-driver/Cargo.toml +++ b/drivers/ax-driver/Cargo.toml @@ -73,7 +73,7 @@ rockchip-sdhci = [ "dep:rdif-clk", "dep:sdhci-host", "dep:sdmmc-protocol", - "sdmmc-protocol/sdio", + "sdmmc-protocol/rdif", ] phytium-mci = [ "block", @@ -81,14 +81,14 @@ phytium-mci = [ "dep:ax-kspin", "dep:phytium-mci-host", "dep:sdmmc-protocol", - "sdmmc-protocol/sdio", + "sdmmc-protocol/rdif", ] k230-sdhci = [ "block", "plat-dyn", "dep:sdhci-host", "dep:sdmmc-protocol", - "sdmmc-protocol/sdio", + "sdmmc-protocol/rdif", ] rockchip-dwmmc = [ "block", @@ -99,7 +99,7 @@ rockchip-dwmmc = [ "dep:dwmmc-host", "dep:rdif-clk", "dep:sdmmc-protocol", - "sdmmc-protocol/sdio", + "sdmmc-protocol/rdif", ] starfive-jh7110-dwmmc = [ "block", diff --git a/drivers/ax-driver/src/block/binding.rs b/drivers/ax-driver/src/block/binding.rs index fec442c490..6bec16ceb7 100644 --- a/drivers/ax-driver/src/block/binding.rs +++ b/drivers/ax-driver/src/block/binding.rs @@ -11,11 +11,12 @@ use core::sync::atomic::{AtomicU64, Ordering}; use ax_errno::{AxError, AxResult}; use ax_kspin::SpinNoIrq; -use dma_api::{ContiguousArray, DeviceDma, DmaDirection}; +use dma_api::{ContiguousArray, CpuDmaBuffer, DeviceDma, DmaDirection}; use log::{error, warn}; use rdif_block::{ - BlkError, Buffer, IQueue, Interface, QueueInfo, Request, RequestFlags, RequestId, RequestOp, - RequestStatus, TransferChunk, TransferPlan, TransferPlanner, TransferRuntimeCaps, + BlkError, Buffer, CompletedRequest, IQueue, Interface, OwnedRequest, QueueHandle, QueueInfo, + Request, RequestFlags, RequestId, RequestOp, RequestPoll, RequestStatus, TransferChunk, + TransferPlan, TransferPlanner, TransferRuntimeCaps, }; use rdrive::{Device, probe::OnProbeError}; @@ -37,12 +38,17 @@ pub struct Block { } struct BlockQueues { - queue: Box, + queue: RuntimeQueue, pool: BlockBufferPool, #[cfg(feature = "irq")] irq_events: Arc, } +enum RuntimeQueue { + Legacy(Box), + Owned(QueueHandle), +} + struct BlockBufferPool { dma: DeviceDma, planner: TransferPlanner, @@ -210,6 +216,25 @@ impl Block { pub fn flush(&mut self) -> AxResult { let mut queues = self.queues.lock(); + if queues.queue.is_owned() { + let request_id = match queues + .queue + .owned_mut() + .ok_or(AxError::BadState)? + .submit_request(OwnedRequest { + op: RequestOp::Flush, + lba: 0, + block_count: 0, + data: None, + flags: RequestFlags::NONE, + }) { + Ok(request_id) => request_id, + Err(err) if err.error == BlkError::NotSupported => return Ok(()), + Err(err) => return Err(map_blk_err_to_ax_err(err.error)), + }; + return queues.poll_owned_until_complete(request_id).map(|_| ()); + } + let mut segments = []; let request = Request { op: RequestOp::Flush, @@ -218,7 +243,12 @@ impl Block { segments: &mut segments, flags: RequestFlags::NONE, }; - let request_id = match queues.queue.submit_request(request) { + let request_id = match queues + .queue + .legacy_mut() + .ok_or(AxError::BadState)? + .submit_request(request) + { Ok(request_id) => request_id, Err(BlkError::NotSupported) => return Ok(()), Err(err) => return Err(map_blk_err_to_ax_err(err)), @@ -235,21 +265,34 @@ impl Block { for chunk in plan { let block_buf = &mut buf[chunk.byte_offset..chunk.byte_offset.saturating_add(chunk.byte_len)]; - let mut dma_buffer = queues.pool.alloc(DmaDirection::FromDevice)?; - dma_buffer.prepare_for_device(0, block_buf.len()); - let mut segments = segments_from_dma(&mut dma_buffer, chunk)?; - let request_id = queues - .queue - .submit_request(Request { + if queues.queue.is_owned() { + let dma_buffer = queues + .pool + .alloc_owned(DmaDirection::FromDevice, block_buf.len())?; + let request_id = queues.submit_owned_request(OwnedRequest { + op: RequestOp::Read, + lba: chunk.lba, + block_count: chunk.block_count, + data: Some(dma_buffer.prepare_for_device()), + flags: RequestFlags::NONE, + })?; + let completed = queues.poll_owned_until_complete(request_id)?; + let completed_dma = completed.data.ok_or(AxError::BadState)?; + completed_dma.copy_from_device_to_slice(block_buf); + } else { + let mut dma_buffer = queues.pool.alloc(DmaDirection::FromDevice)?; + dma_buffer.prepare_for_device(0, block_buf.len()); + let mut segments = segments_from_dma(&mut dma_buffer, chunk)?; + let request_id = queues.submit_legacy_request(Request { op: RequestOp::Read, lba: chunk.lba, block_count: chunk.block_count, segments: &mut segments, flags: RequestFlags::NONE, - }) - .map_err(map_blk_err_to_ax_err)?; - queues.poll_until_complete(request_id)?; - dma_buffer.copy_from_device_to_slice(block_buf); + })?; + queues.poll_until_complete(request_id)?; + dma_buffer.copy_from_device_to_slice(block_buf); + } } Ok(()) } @@ -263,20 +306,33 @@ impl Block { for chunk in plan { let block_buf = &buf[chunk.byte_offset..chunk.byte_offset.saturating_add(chunk.byte_len)]; - let mut dma_buffer = queues.pool.alloc(DmaDirection::ToDevice)?; - dma_buffer.copy_to_device_from_slice(block_buf); - let mut segments = segments_from_dma(&mut dma_buffer, chunk)?; - let request_id = queues - .queue - .submit_request(Request { + if queues.queue.is_owned() { + let mut dma_buffer = queues + .pool + .alloc_owned(DmaDirection::ToDevice, block_buf.len())?; + dma_buffer.copy_to_device_from_slice(block_buf); + let request_id = queues.submit_owned_request(OwnedRequest { + op: RequestOp::Write, + lba: chunk.lba, + block_count: chunk.block_count, + data: Some(dma_buffer.prepare_for_device()), + flags: RequestFlags::NONE, + })?; + let completed = queues.poll_owned_until_complete(request_id)?; + drop(completed.data); + } else { + let mut dma_buffer = queues.pool.alloc(DmaDirection::ToDevice)?; + dma_buffer.copy_to_device_from_slice(block_buf); + let mut segments = segments_from_dma(&mut dma_buffer, chunk)?; + let request_id = queues.submit_legacy_request(Request { op: RequestOp::Write, lba: chunk.lba, block_count: chunk.block_count, segments: &mut segments, flags: RequestFlags::NONE, - }) - .map_err(map_blk_err_to_ax_err)?; - queues.poll_until_complete(request_id)?; + })?; + queues.poll_until_complete(request_id)?; + } } Ok(()) } @@ -328,7 +384,7 @@ impl RdifBlockDevice { impl BlockQueues { fn new( - queue: Box, + queue: RuntimeQueue, #[cfg(feature = "irq")] irq_events: Arc, ) -> AxResult { let info = queue.info(); @@ -341,7 +397,11 @@ impl BlockQueues { Ok(Self { queue, pool: BlockBufferPool { - dma: DeviceDma::new(info.limits.dma_mask, axklib::dma::op()), + dma: DeviceDma::new( + info.limits.dma_domain, + info.limits.dma_mask, + axklib::dma::op(), + ), planner, size: layout.size(), align: layout.align(), @@ -351,13 +411,31 @@ impl BlockQueues { }) } + fn submit_legacy_request(&mut self, request: Request<'_>) -> AxResult { + self.queue + .legacy_mut() + .ok_or(AxError::BadState)? + .submit_request(request) + .map_err(map_blk_err_to_ax_err) + } + + fn submit_owned_request(&mut self, request: OwnedRequest) -> AxResult { + self.queue + .owned_mut() + .ok_or(AxError::BadState)? + .submit_request(request) + .map_err(|err| map_blk_err_to_ax_err(err.error)) + } + fn poll_until_complete(&mut self, request: RequestId) -> AxResult { loop { - match self + let status = self .queue + .legacy_mut() + .ok_or(AxError::BadState)? .poll_request(request) - .map_err(map_blk_err_to_ax_err)? - { + .map_err(map_blk_err_to_ax_err)?; + match status { RequestStatus::Complete => { #[cfg(feature = "irq")] let _ = self.irq_events.take_queue(self.queue.id()); @@ -373,6 +451,32 @@ impl BlockQueues { } } } + + fn poll_owned_until_complete(&mut self, request: RequestId) -> AxResult { + loop { + let status = self + .queue + .owned_mut() + .ok_or(AxError::BadState)? + .poll_request(request) + .map_err(|err| map_blk_err_to_ax_err(err.into()))?; + match status { + RequestPoll::Ready(completed) => { + #[cfg(feature = "irq")] + let _ = self.irq_events.take_queue(self.queue.id()); + completed.result.map_err(map_blk_err_to_ax_err)?; + return Ok(completed); + } + RequestPoll::Pending => { + #[cfg(feature = "irq")] + if self.irq_events.take_queue(self.queue.id()) { + continue; + } + core::hint::spin_loop(); + } + } + } + } } impl BlockBufferPool { @@ -382,6 +486,52 @@ impl BlockBufferPool { .map_err(BlkError::from) .map_err(map_blk_err_to_ax_err) } + + fn alloc_owned(&self, direction: DmaDirection, len: usize) -> AxResult { + CpuDmaBuffer::new_zero( + &self.dma, + core::num::NonZeroUsize::new(len).ok_or(AxError::BadState)?, + self.align, + direction, + ) + .map_err(BlkError::from) + .map_err(map_blk_err_to_ax_err) + } +} + +impl RuntimeQueue { + #[cfg(feature = "irq")] + fn id(&self) -> usize { + match self { + Self::Legacy(queue) => queue.id(), + Self::Owned(queue) => queue.id(), + } + } + + fn info(&self) -> QueueInfo { + match self { + Self::Legacy(queue) => queue.info(), + Self::Owned(queue) => queue.info(), + } + } + + fn is_owned(&self) -> bool { + matches!(self, Self::Owned(_)) + } + + fn legacy_mut(&mut self) -> Option<&mut dyn IQueue> { + match self { + Self::Legacy(queue) => Some(&mut **queue), + Self::Owned(_) => None, + } + } + + fn owned_mut(&mut self) -> Option<&mut QueueHandle> { + match self { + Self::Legacy(_) => None, + Self::Owned(queue) => Some(queue), + } + } } impl TryFrom> for Block { @@ -393,7 +543,11 @@ impl TryFrom> for Block { let info = dev.info.clone(); let irq = info.irq_num(); let mut interface = dev.interface.take().ok_or(AxError::BadState)?; - let queue = interface.create_queue().ok_or(AxError::BadState)?; + let queue = interface + .create_owned_queue() + .map(RuntimeQueue::Owned) + .or_else(|| interface.create_queue().map(RuntimeQueue::Legacy)) + .ok_or(AxError::BadState)?; #[cfg(feature = "irq")] let irq_events = Arc::new(BlockIrqEvents::default()); let queues = BlockQueues::new( @@ -662,7 +816,10 @@ mod tests { use dma_api::{ DeviceDma, DmaAllocHandle, DmaConstraints, DmaDirection, DmaError, DmaMapHandle, DmaOp, }; - use rdif_block::{DeviceInfo, DriverGeneric, QueueLimits, validate_request}; + use rdif_block::{ + DeviceInfo, DriverGeneric, IQueueOwned, PollError, QueueLimits, SubmitError, + validate_request, + }; use super::*; @@ -897,6 +1054,97 @@ mod tests { } } + struct OwnedRecordingQueue { + info: QueueInfo, + log: &'static RequestLog, + pending: Option, + } + + impl OwnedRecordingQueue { + fn new(log: &'static RequestLog, limits: QueueLimits) -> Self { + Self { + info: QueueInfo { + id: 0, + device: DeviceInfo { + name: Some("owned-recording-block"), + ..DeviceInfo::new(64, 512) + }, + limits, + }, + log, + pending: None, + } + } + } + + impl IQueueOwned for OwnedRecordingQueue { + fn id(&self) -> usize { + self.info.id + } + + fn info(&self) -> QueueInfo { + self.info + } + + fn submit_request(&mut self, request: OwnedRequest) -> Result { + if let Err(err) = rdif_block::validate_owned_request(self.info, &request) { + return Err(SubmitError::new(err, request)); + } + let id = RequestId::new(self.log.snapshot().len()); + let op = request.op; + let lba = request.lba; + let block_count = request.block_count; + let flags = request.flags; + let data_len = request.data_len(); + let mut completed_data = request.data.map(|data| data.into_cpu_buffer()); + if op == RequestOp::Read { + let Some(buffer) = completed_data.as_mut() else { + return Err(SubmitError::new( + BlkError::InvalidRequest, + OwnedRequest { + op, + lba, + block_count, + data: None, + flags, + }, + )); + }; + unsafe { + buffer.as_mut_slice_cpu().fill(block_count as u8); + } + } + self.log.push(SubmittedRequest { + op, + lba, + block_count, + data_len, + segment_lengths: alloc::vec![data_len], + }); + let completed_data = completed_data.map(|buffer| { + let in_flight = unsafe { buffer.prepare_for_device().into_in_flight() }; + unsafe { in_flight.complete_after_quiesce() } + }); + self.pending = Some(CompletedRequest::new(id, Ok(()), completed_data)); + Ok(id) + } + + fn poll_request(&mut self, _request: RequestId) -> Result { + Ok(self + .pending + .take() + .map(RequestPoll::Ready) + .unwrap_or(RequestPoll::Pending)) + } + + fn cancel_request(&mut self, _request: RequestId) -> Result { + self.pending + .take() + .map(RequestPoll::Ready) + .ok_or(PollError::UnknownRequest) + } + } + // SAFETY: The queue records request metadata synchronously and never // accesses request segments after `submit_request` returns. unsafe impl IQueue for RecordingQueue { @@ -940,9 +1188,9 @@ mod tests { irq_handler: None, interface: Box::new(TestInterface), queues: SpinNoIrq::new(BlockQueues { - queue, + queue: RuntimeQueue::Legacy(queue), pool: BlockBufferPool { - dma: DeviceDma::new(u64::MAX, dma), + dma: DeviceDma::new_legacy(u64::MAX, dma), planner, size: layout.size(), align: layout.align(), @@ -970,6 +1218,7 @@ mod tests { let log = Box::leak(Box::::default()); let limits = QueueLimits { dma_mask: u64::MAX, + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_alignment: 4096, max_inflight: 1, max_blocks_per_request: 8, @@ -1019,6 +1268,7 @@ mod tests { let log = Box::leak(Box::::default()); let limits = QueueLimits { dma_mask: u64::MAX, + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_alignment: 4096, max_inflight: 1, max_blocks_per_request: 8, @@ -1070,6 +1320,70 @@ mod tests { ); } + #[test] + fn owned_queue_read_returns_completed_dma_to_caller_buffer() { + let dma = Box::leak(Box::::default()); + let log = Box::leak(Box::::default()); + let limits = QueueLimits { + dma_mask: u64::MAX, + dma_domain: dma_api::DmaDomainId::legacy_global(), + dma_alignment: 4096, + max_inflight: 1, + max_blocks_per_request: 8, + max_segments: 1, + max_segment_size: 4096, + supported_flags: RequestFlags::NONE, + supports_flush: false, + supports_discard: false, + supports_write_zeroes: false, + }; + let queue = QueueHandle::new(Box::new(OwnedRecordingQueue::new(log, limits))); + let info = queue.info(); + let planner = block_transfer_planner(info).unwrap(); + let layout = block_buffer_layout(info, planner.chunk_size()).unwrap(); + let mut queues = BlockQueues { + queue: RuntimeQueue::Owned(queue), + pool: BlockBufferPool { + dma: DeviceDma::new_legacy(u64::MAX, dma), + planner, + size: layout.size(), + align: layout.align(), + }, + #[cfg(feature = "irq")] + irq_events: Arc::new(BlockIrqEvents::default()), + }; + let mut buf = [0_u8; 4096]; + + let dma_buffer = queues + .pool + .alloc_owned(DmaDirection::FromDevice, buf.len()) + .unwrap(); + let request_id = queues + .submit_owned_request(OwnedRequest { + op: RequestOp::Read, + lba: 4, + block_count: 8, + data: Some(dma_buffer.prepare_for_device()), + flags: RequestFlags::NONE, + }) + .unwrap(); + let completed = queues.poll_owned_until_complete(request_id).unwrap(); + let completed_dma = completed.data.unwrap(); + completed_dma.copy_from_device_to_slice(&mut buf); + + assert_eq!(buf, [8; 4096]); + assert_eq!( + log.snapshot(), + [SubmittedRequest { + op: RequestOp::Read, + lba: 4, + block_count: 8, + data_len: 4096, + segment_lengths: alloc::vec![4096], + }] + ); + } + #[test] fn block_transfer_planner_caps_large_finite_segments() { let info = QueueInfo { @@ -1080,6 +1394,7 @@ mod tests { }, limits: QueueLimits { dma_mask: u64::MAX, + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_alignment: 4096, max_inflight: 1, max_blocks_per_request: 4096, @@ -1122,6 +1437,7 @@ mod tests { }, limits: QueueLimits { dma_mask: u64::MAX, + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_alignment: 4096, max_inflight: 1, max_blocks_per_request: u32::MAX, @@ -1146,6 +1462,7 @@ mod tests { let log = Box::leak(Box::::default()); let limits = QueueLimits { dma_mask: u64::MAX, + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_alignment: 4096, max_inflight: 1, max_blocks_per_request: 8, diff --git a/drivers/ax-driver/src/block/k230_sdhci.rs b/drivers/ax-driver/src/block/k230_sdhci.rs index 3d807825a1..9ee13f4047 100644 --- a/drivers/ax-driver/src/block/k230_sdhci.rs +++ b/drivers/ax-driver/src/block/k230_sdhci.rs @@ -20,24 +20,16 @@ use rdrive::{ probe::OnProbeError, register::{FdtInfo, ProbeFdt}, }; -use sdhci_host::Sdhci; +use sdhci_host::{Sdhci, rdif as sdhci_rdif}; use sdmmc_protocol::{ Error, OperationPoll, error::Phase, - sdio::{CardInfo, CardInitPreference, SdioInitScratch, SdioSdmmc}, + sdio::{CardInfo, CardInitPreference, SdioHost2Adapter, SdioInitScratch, SdioSdmmc}, }; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; - -const SDHCI_POWER_330: u8 = 0x0e; +use crate::{block::ProbeFdtBlock, mmio::iomap}; -type K230Sdhci = SdioSdmmc; +type K230Sdhci = SdioSdmmc>; crate::model_register!( name: "K230 SDHCI", @@ -73,15 +65,11 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { let mmio_base = iomap(base_reg.address as usize, mmio_size as usize)?; let mut host = unsafe { Sdhci::new(mmio_base) }; - info!("k230-sdhci: reset controller"); - host.reset_all() - .map_err(|e| init_error(base_reg.address, mmio_size, e))?; - host.set_power(SDHCI_POWER_330); - host.enable_interrupts(); - host.set_dma(axklib::dma::device_with_mask(u32::MAX as u64)); - - info!("k230-sdhci: initialize card"); - let mut card = SdioSdmmc::new(host); + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); + + info!("k230-sdhci: initialize card through native host2 bus ops"); + let mut card = SdioSdmmc::new_host2(host); let card_info = poll_card_init(&mut card, card_init_preference(info)) .map_err(|e| card_init_error(base_reg.address, mmio_size, e))?; info!( @@ -96,10 +84,14 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(card); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::dma("k230-sdhci", card_info.capacity_blocks.unwrap_or(0), false), + let dev = sdhci_rdif::device( + card, + sdhci_rdif::dma_config( + "k230-sdhci", + card_info.capacity_blocks.unwrap_or(0), + false, + dma, + ), ); let irq = probe.register_block(dev)?; info!("k230-sdhci block device registered irq={:?}", irq); diff --git a/drivers/ax-driver/src/block/mod.rs b/drivers/ax-driver/src/block/mod.rs index 852b7f7533..d66c54ce7e 100644 --- a/drivers/ax-driver/src/block/mod.rs +++ b/drivers/ax-driver/src/block/mod.rs @@ -1,13 +1,5 @@ mod binding; -#[cfg(any( - feature = "k230-sdhci", - feature = "phytium-mci", - feature = "rockchip-dwmmc", - feature = "rockchip-sdhci", - feature = "starfive-jh7110-dwmmc" -))] -pub(crate) mod sdmmc; #[allow(unused)] mod shared; diff --git a/drivers/ax-driver/src/block/phytium_mci.rs b/drivers/ax-driver/src/block/phytium_mci.rs index 26c9e93f7e..fbb584b486 100644 --- a/drivers/ax-driver/src/block/phytium_mci.rs +++ b/drivers/ax-driver/src/block/phytium_mci.rs @@ -2,7 +2,7 @@ use alloc::format; use core::time::Duration; use log::{info, warn}; -use phytium_mci_host::PhytiumMci; +use phytium_mci_host::{PhytiumMci, rdif as phytium_rdif}; use rdrive::{ probe::OnProbeError, register::{FdtInfo, ProbeFdt}, @@ -10,18 +10,17 @@ use rdrive::{ use sdmmc_protocol::{ Error, OperationPoll, error::Phase, - sdio::{CardInfo, CardInitPreference, SdioInitScratch, SdioSdmmc}, + sdio::{CardInfo, CardInitPreference, SdioHost2Adapter, SdioInitScratch, SdioSdmmc}, }; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap}; + +type PhytiumSdMmc = SdioSdmmc>; -type PhytiumSdMmc = SdioSdmmc; +// Phytium MCI exposes a lock-free top-half IRQ handle: interrupt context +// acknowledges/caches raw and IDMAC status, while task-side RDIF polling +// advances the request and releases DMA buffers. +const PHYTIUM_MCI_IRQ_DRIVEN: bool = true; crate::model_register!( name: "Phytium MCI", @@ -57,12 +56,11 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { let mmio_base = iomap(base_reg.address as usize, mmio_size as usize)?; let mut host = unsafe { PhytiumMci::new(mmio_base) }; - info!("phytium-mci: reset controller"); - host.reset_and_init() - .map_err(|e| init_error(base_reg.address, mmio_size, e))?; + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); - info!("phytium-mci: initialize card"); - let mut card = SdioSdmmc::new(host); + info!("phytium-mci: initialize card through native host2 bus ops"); + let mut card = SdioSdmmc::new_host2(host); card.set_sd_uhs_selection_enabled(false); let preference = card_init_preference(info); let card_info = poll_card_init(&mut card, preference).map_err(|e| { @@ -81,10 +79,14 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(card); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::fifo("phytium-mci", card_info.capacity_blocks.unwrap_or(0), false), + let dev = phytium_rdif::device( + card, + phytium_rdif::dma_config( + "phytium-mci", + card_info.capacity_blocks.unwrap_or(0), + PHYTIUM_MCI_IRQ_DRIVEN, + dma, + ), ); let irq = probe.register_block(dev)?; info!("phytium-mci block device registered irq={:?}", irq); @@ -173,4 +175,17 @@ mod tests { assert!(!is_absent_card_init_error(err)); } + + #[test] + fn phytium_mci_block_io_uses_dma_config_with_irq_completion() { + let config = phytium_rdif::dma_config( + "phytium-mci", + 8, + PHYTIUM_MCI_IRQ_DRIVEN, + axklib::dma::device_with_mask(u32::MAX as u64), + ); + + assert!(config.uses_dma()); + assert!(config.irq_driven); + } } diff --git a/drivers/ax-driver/src/block/rockchip/sdhci_rk3568.rs b/drivers/ax-driver/src/block/rockchip/sdhci_rk3568.rs index 71fdbefa26..776778fe60 100644 --- a/drivers/ax-driver/src/block/rockchip/sdhci_rk3568.rs +++ b/drivers/ax-driver/src/block/rockchip/sdhci_rk3568.rs @@ -22,23 +22,20 @@ use rdrive::{ probe::OnProbeError, register::{FdtInfo, ProbeFdt}, }; -use sdhci_host::{HostClock, Sdhci}; +use sdhci_host::{HostClock, HostResetHook, Sdhci, rdif as sdhci_rdif}; use sdmmc_protocol::{ Error, OperationPoll, error::{ErrorContext, Phase}, - sdio::{CardInfo, CardInitPreference, SdioInitScratch, SdioSdmmc}, + sdio::{CardInfo, CardInitPreference, SdioHost2Adapter, SdioInitScratch, SdioSdmmc}, }; use spin::Once; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap}; -const SDHCI_POWER_330: u8 = 0x0e; +// RK3568 DWCMSHC uses the same SDHCI completion interrupt path as RK3588: +// the hard IRQ acknowledges/caches controller status and task-side RDIF +// polling consumes the completion. +const ROCKCHIP_RK3568_SDHCI_IRQ_DRIVEN: bool = true; const DWCMSHC_P_VENDOR_AREA1: usize = 0xe8; const DWCMSHC_AREA1_MASK: u16 = 0x0fff; @@ -87,10 +84,12 @@ const PHY_DLL_CTRL_ENABLE: u8 = 0x1; const PHY_DLL_CNFG2_JUMPSTEP: u8 = 0x0a; static SDHCI_CLOCK: RockchipSdhciClock = RockchipSdhciClock; +static SDHCI_RESET_HOOK: RockchipSdhciResetHook = RockchipSdhciResetHook; -type RockchipSdhci = SdioSdmmc; +type RockchipSdhci = SdioSdmmc>; struct RockchipSdhciClock; +struct RockchipSdhciResetHook; impl HostClock for RockchipSdhciClock { fn set_clock(&self, target_hz: u32) -> Result<(), Error> { @@ -98,6 +97,12 @@ impl HostClock for RockchipSdhciClock { } } +impl HostResetHook for RockchipSdhciResetHook { + fn after_reset(&self, host: &mut Sdhci) -> Result<(), Error> { + init_dwcmshc_after_reset(host) + } +} + crate::model_register!( name: "Rockchip RK3568 sdhci", level: ProbeLevel::PostKernel, @@ -140,16 +145,12 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { } else { warn!("rockchip-rk3568-sdhci: no core clock found; using SDHCI internal clock divider"); } - info!("rockchip-rk3568-sdhci: reset controller"); - host.reset_all() - .map_err(|e| init_error(base_reg.address, mmio_size, e))?; - init_dwcmshc_after_reset(mmio_base); - host.set_power(SDHCI_POWER_330); - host.enable_interrupts(); - host.set_dma(axklib::dma::device_with_mask(u32::MAX as u64)); - - info!("rockchip-rk3568-sdhci: initialize card"); - let mut card = SdioSdmmc::new(host); + host.set_reset_hook(&SDHCI_RESET_HOOK); + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); + + info!("rockchip-rk3568-sdhci: initialize card through native host2 bus ops"); + let mut card = SdioSdmmc::new_host2(host); let card_info = poll_card_init_mmc(&mut card) .map_err(|e| card_init_error(base_reg.address, mmio_size, e))?; info!( @@ -164,13 +165,13 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(card); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::fifo( + let dev = sdhci_rdif::device( + card, + sdhci_rdif::dma_config( "rockchip-rk3568-sdhci", card_info.capacity_blocks.unwrap_or(0), - true, + ROCKCHIP_RK3568_SDHCI_IRQ_DRIVEN, + dma, ), ); let irq = probe.register_block(dev)?; @@ -200,7 +201,8 @@ fn poll_card_init_mmc(card: &mut RockchipSdhci) -> Result { } } -fn init_dwcmshc_after_reset(base: NonNull) { +fn init_dwcmshc_after_reset(host: &mut Sdhci) -> Result<(), Error> { + let base = NonNull::new(host.mmio_base() as *mut u8).ok_or(Error::InvalidArgument)?; let area1 = vendor_area1(base); // Match Linux rk35xx reset/set_clock setup for identification speed: @@ -237,6 +239,7 @@ fn init_dwcmshc_after_reset(base: NonNull) { "rockchip-rk3568-sdhci: dwcmshc vendor init area1={:#x}", area1 ); + Ok(()) } fn init_dwcmshc_phy_3v3(base: NonNull) { @@ -383,19 +386,30 @@ mod tests { use super::*; #[test] - fn rk3568_block_io_uses_fifo_config() { - let config = SdmmcBlockConfig::fifo("rockchip-rk3568-sdhci", 8, true); + fn rk3568_block_io_uses_dma_config_with_irq_completion() { + let config = sdhci_rdif::dma_config( + "rockchip-rk3568-sdhci", + 8, + ROCKCHIP_RK3568_SDHCI_IRQ_DRIVEN, + axklib::dma::device_with_mask(u32::MAX as u64), + ); - assert!(!config.use_dma); + assert!(config.uses_dma()); + assert!(config.irq_driven); } #[test] - fn rk3568_fifo_queue_limits_single_block_requests() { - let config = SdmmcBlockConfig::fifo("rockchip-rk3568-sdhci", 8, true); - let limits = crate::block::sdmmc::queue_limits(&config, u32::MAX as u64); + fn rk3568_dma_queue_limits_multi_block_requests() { + let config = sdhci_rdif::dma_config( + "rockchip-rk3568-sdhci", + 8, + true, + axklib::dma::device_with_mask(u32::MAX as u64), + ); + let limits = sdmmc_protocol::rdif::queue_limits(&config, u32::MAX as u64); - assert_eq!(limits.max_blocks_per_request, 1); - assert_eq!(limits.max_segment_size, crate::block::sdmmc::BLOCK_SIZE); + assert!(limits.max_blocks_per_request > 1); + assert!(limits.max_segment_size > sdmmc_protocol::rdif::BLOCK_SIZE); assert_eq!(limits.max_segments, 1); } } diff --git a/drivers/ax-driver/src/block/rockchip_mmc.rs b/drivers/ax-driver/src/block/rockchip_mmc.rs index dedb26e8e4..a5ea372107 100644 --- a/drivers/ax-driver/src/block/rockchip_mmc.rs +++ b/drivers/ax-driver/src/block/rockchip_mmc.rs @@ -22,26 +22,23 @@ use rdrive::{ probe::OnProbeError, register::{FdtInfo, ProbeFdt}, }; -use sdhci_host::{HostClock, Sdhci}; +use sdhci_host::{HostClock, Sdhci, rdif as sdhci_rdif}; use sdmmc_protocol::{ Error, OperationPoll, error::{ErrorContext, Phase}, - sdio::{CardInfo, CardInitPreference, SdioInitScratch, SdioSdmmc}, + sdio::{CardInfo, CardInitPreference, SdioHost2Adapter, SdioInitScratch, SdioSdmmc}, }; use spin::Once; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap}; -const SDHCI_POWER_330: u8 = 0x0e; +// RK3588 DWCMSHC follows Linux's normal SDHCI completion path: command/data +// status is acknowledged in the hard IRQ and task context advances the RDIF +// submit/poll queue. +const ROCKCHIP_SDHCI_IRQ_DRIVEN: bool = true; static SDHCI_CLOCK: RockchipSdhciClock = RockchipSdhciClock; -type RockchipSdhci = SdioSdmmc; +type RockchipSdhci = SdioSdmmc>; struct RockchipSdhciClock; @@ -93,15 +90,11 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { } else { warn!("rockchip-sdhci: no core clock found; using SDHCI internal clock divider"); } - info!("rockchip-sdhci: reset controller"); - host.reset_all() - .map_err(|e| init_error(base_reg.address, mmio_size, e))?; - host.set_power(SDHCI_POWER_330); - host.enable_interrupts(); - host.set_dma(axklib::dma::device_with_mask(u32::MAX as u64)); - - info!("rockchip-sdhci: initialize card"); - let mut card = SdioSdmmc::new(host); + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); + + info!("rockchip-sdhci: initialize card through native host2 bus ops"); + let mut card = SdioSdmmc::new_host2(host); let card_info = poll_card_init_mmc(&mut card) .map_err(|e| card_init_error(base_reg.address, mmio_size, e))?; info!( @@ -116,20 +109,27 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(card); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::dma( - "rockchip-sdhci", - card_info.capacity_blocks.unwrap_or(0), - true, - ), + let dev = sdhci_rdif::device( + card, + rockchip_sdhci_rdif_config(card_info.capacity_blocks.unwrap_or(0), dma), ); let irq = probe.register_block(dev)?; info!("rockchip-sdhci block device registered irq={:?}", irq); Ok(()) } +fn rockchip_sdhci_rdif_config( + capacity_blocks: u64, + dma: dma_api::DeviceDma, +) -> sdhci_rdif::BlockConfig { + sdhci_rdif::dma_config( + "rockchip-sdhci", + capacity_blocks, + ROCKCHIP_SDHCI_IRQ_DRIVEN, + dma, + ) +} + fn poll_card_init_mmc(card: &mut RockchipSdhci) -> Result { let mut scratch = SdioInitScratch::new(); let mut request = @@ -235,3 +235,73 @@ struct ClkDev { inner: Device, id: ClockId, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rk3588_block_io_uses_adma_config_with_irq_completion() { + let config = rockchip_sdhci_rdif_config(8, test_dma()); + + assert_eq!(config.name, "rockchip-sdhci"); + assert_eq!(config.capacity_blocks, 8); + assert!(config.uses_dma()); + assert!(config.irq_driven); + } + + #[test] + fn rk3588_adma_queue_limits_expose_sdhci_window() { + let config = rockchip_sdhci_rdif_config(8, test_dma()); + let limits = sdmmc_protocol::rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, sdhci_host::ADMA2_MAX_BLOCKS); + assert_eq!(limits.max_segment_size, sdhci_host::ADMA2_MAX_TRANSFER_SIZE); + assert_eq!(limits.max_segments, 1); + } + + fn test_dma() -> dma_api::DeviceDma { + dma_api::DeviceDma::new_legacy(u32::MAX as u64, &TEST_DMA) + } + + struct TestDma; + static TEST_DMA: TestDma = TestDma; + + impl dma_api::DmaOp for TestDma { + fn page_size(&self) -> usize { + sdmmc_protocol::rdif::BLOCK_SIZE + } + + unsafe fn alloc_contiguous( + &self, + _constraints: dma_api::DmaConstraints, + _layout: core::alloc::Layout, + ) -> Option { + None + } + + unsafe fn dealloc_contiguous(&self, _handle: dma_api::DmaAllocHandle) {} + + unsafe fn alloc_coherent( + &self, + _constraints: dma_api::DmaConstraints, + _layout: core::alloc::Layout, + ) -> Option { + None + } + + unsafe fn dealloc_coherent(&self, _handle: dma_api::DmaAllocHandle) {} + + unsafe fn map_streaming( + &self, + _constraints: dma_api::DmaConstraints, + _addr: core::ptr::NonNull, + _size: core::num::NonZeroUsize, + _direction: dma_api::DmaDirection, + ) -> Result { + Err(dma_api::DmaError::NoMemory) + } + + unsafe fn unmap_streaming(&self, _handle: dma_api::DmaMapHandle) {} + } +} diff --git a/drivers/ax-driver/src/block/rockchip_sd.rs b/drivers/ax-driver/src/block/rockchip_sd.rs index 957f43dd7f..49a81548ea 100644 --- a/drivers/ax-driver/src/block/rockchip_sd.rs +++ b/drivers/ax-driver/src/block/rockchip_sd.rs @@ -15,7 +15,7 @@ use alloc::format; use core::time::Duration; -use dwmmc_host::DwMmc; +use dwmmc_host::{DwMmc, rdif as dwmmc_rdif}; use log::{info, warn}; use rdif_clk::ClockId; use rdrive::{ @@ -25,17 +25,10 @@ use rdrive::{ use sdmmc_protocol::{ Error, OperationPoll, error::Phase, - sdio::{CardInfo, SdioInitScratch, SdioSdmmc}, + sdio::{CardInfo, SdioHost2Adapter, SdioInitScratch, SdioSdmmc}, }; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, - soc::scmi, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap, soc::scmi}; const DWMMC_STABLE_REFERENCE_CLOCK: u32 = 50_000_000; const ENABLE_SD_SPEED_SELECTION: bool = true; @@ -48,7 +41,7 @@ const RK3588_SDMMC_DRV_PHASE_DEG: u32 = 90; const RK3588_SDMMC_SAMPLE_PHASE_DEG: u32 = 0; const RK3588_SDMMC_SAMPLE_PHASE_CANDIDATES: [u32; 8] = [0, 45, 90, 135, 180, 225, 270, 315]; -type RockchipDwMmc = SdioSdmmc; +type RockchipDwMmc = SdioSdmmc>; mod phase; @@ -104,13 +97,11 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { CRU rate" ); } - info!("rockchip-dwmmc: reset controller"); - host.reset_and_init() - .map_err(|e| init_error(base_reg.address, mmio_size, e))?; - host.set_dma(axklib::dma::device_with_mask(u32::MAX as u64)); + let dma = axklib::dma::device_with_mask(u32::MAX as u64); + host.set_dma(dma.clone()); - info!("rockchip-dwmmc: initialize card"); - let mut sd = SdioSdmmc::new(host); + info!("rockchip-dwmmc: initialize card through native host2 bus ops"); + let mut sd = SdioSdmmc::new_host2(host); sd.set_sd_speed_selection_enabled(ENABLE_SD_SPEED_SELECTION); let card_info = poll_card_init(&mut sd).map_err(|e| { warn!("rockchip-dwmmc: card init failed: {:?}", e); @@ -134,10 +125,14 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { tune_rk3588_sdmmc_sample_phase(&mut sd, reference_clock); } - let raw = SharedDriver::new(sd); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::dma("rockchip-sd", card_info.capacity_blocks.unwrap_or(0), true), + let dev = dwmmc_rdif::device( + sd, + dwmmc_rdif::dma_config( + "rockchip-sd", + card_info.capacity_blocks.unwrap_or(0), + true, + dma, + ), ); let irq = probe.register_block(dev)?; info!("rockchip-sd block device registered irq={:?}", irq); diff --git a/drivers/ax-driver/src/block/rockchip_sd/phase.rs b/drivers/ax-driver/src/block/rockchip_sd/phase.rs index 7e2e37052a..d13f1255fe 100644 --- a/drivers/ax-driver/src/block/rockchip_sd/phase.rs +++ b/drivers/ax-driver/src/block/rockchip_sd/phase.rs @@ -2,14 +2,14 @@ use core::ptr::NonNull; use log::{info, warn}; use rdrive::{probe::OnProbeError, register::FdtInfo}; -use sdmmc_protocol::{DataCommandPoll, Error}; +use sdmmc_protocol::{DataCommandPoll, Error, rdif::BLOCK_SIZE}; use super::{ RK3588_CRU_BASE, RK3588_CRU_SIZE, RK3588_SDMMC_CON0, RK3588_SDMMC_CON1, RK3588_SDMMC_DRV_PHASE_DEG, RK3588_SDMMC_PHASE_SHIFT, RK3588_SDMMC_SAMPLE_PHASE_CANDIDATES, RK3588_SDMMC_SAMPLE_PHASE_DEG, RockchipDwMmc, }; -use crate::{block::sdmmc::BLOCK_SIZE, mmio::iomap}; +use crate::mmio::iomap; pub(super) fn init_rk3588_sdmmc_phase( info: &FdtInfo<'_>, diff --git a/drivers/ax-driver/src/block/sdmmc.rs b/drivers/ax-driver/src/block/sdmmc.rs deleted file mode 100644 index 115e86a905..0000000000 --- a/drivers/ax-driver/src/block/sdmmc.rs +++ /dev/null @@ -1,1084 +0,0 @@ -use alloc::{boxed::Box, vec, vec::Vec}; -use core::{ - marker::PhantomData, - num::NonZeroUsize, - ptr::NonNull, - sync::atomic::{AtomicBool, Ordering}, -}; - -use dma_api::DeviceDma; -use log::warn; -use rdrive::DriverGeneric; -use sdmmc_protocol::{ - BlockPoll, BlockRequestId, Error, - sdio::{SdioHost, SdioIrqHandle, SdioIrqHost, SdioSdmmc, block_queue_ready_from_host_event}, -}; - -use crate::block::SharedDriver; - -pub(crate) const BLOCK_SIZE: usize = 512; - -#[derive(Clone, Copy)] -pub(crate) struct SdmmcBlockConfig { - pub name: &'static str, - pub capacity_blocks: u64, - pub dma_mask: u64, - pub max_blocks_per_request: u32, - pub max_segment_size: usize, - pub irq_driven: bool, - pub use_dma: bool, -} - -impl SdmmcBlockConfig { - #[cfg(any( - feature = "k230-sdhci", - feature = "rockchip-dwmmc", - feature = "rockchip-sdhci", - test - ))] - pub(crate) fn dma(name: &'static str, capacity_blocks: u64, irq_driven: bool) -> Self { - Self { - name, - capacity_blocks, - dma_mask: u32::MAX as u64, - max_blocks_per_request: u16::MAX as u32 + 1, - max_segment_size: usize::MAX, - irq_driven, - use_dma: true, - } - } - - #[cfg(any( - feature = "rockchip-sdhci", - feature = "phytium-mci", - feature = "starfive-jh7110-dwmmc", - test - ))] - pub(crate) fn fifo(name: &'static str, capacity_blocks: u64, irq_driven: bool) -> Self { - Self { - name, - capacity_blocks, - dma_mask: u32::MAX as u64, - max_blocks_per_request: 1, - max_segment_size: BLOCK_SIZE, - irq_driven, - use_dma: false, - } - } -} - -pub(crate) trait SdmmcBlockHost: SdioIrqHost + Send + Sync + 'static { - type Request: Send + 'static; - type Slot: Default + Send + 'static; - - fn submit_read_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result; - - fn submit_write_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result; - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - slot: &mut Self::Slot, - ) -> Result; - - fn request_id(request: &Self::Request) -> BlockRequestId; -} - -pub(crate) struct SdmmcBlockDevice -where - H: SdmmcBlockHost, -{ - raw: Option>>, - irq_handle: ::IrqHandle, - config: SdmmcBlockConfig, - irq_enabled: AtomicBool, - queue_created: bool, - irq_handler_taken: bool, -} - -impl SdmmcBlockDevice -where - H: SdmmcBlockHost, -{ - pub(crate) fn new(raw: SharedDriver>, config: SdmmcBlockConfig) -> Self { - let irq_handle = raw.with_mut(|raw| raw.host().irq_handle()); - Self { - raw: Some(raw), - irq_handle, - config, - irq_enabled: AtomicBool::new(false), - queue_created: false, - irq_handler_taken: false, - } - } - - fn queue_limits_with_mask(&self, dma_mask: u64) -> rdif_block::QueueLimits { - queue_limits(&self.config, dma_mask) - } -} - -impl DriverGeneric for SdmmcBlockDevice -where - H: SdmmcBlockHost, -{ - fn name(&self) -> &str { - self.config.name - } -} - -impl rdif_block::Interface for SdmmcBlockDevice -where - H: SdmmcBlockHost, -{ - fn device_info(&self) -> rdif_block::DeviceInfo { - device_info(&self.config) - } - - fn queue_limits(&self) -> rdif_block::QueueLimits { - self.queue_limits_with_mask(self.config.dma_mask) - } - - fn create_queue(&mut self) -> Option> { - if self.queue_created { - return None; - } - self.raw.clone().as_ref().map(|dev| { - self.queue_created = true; - Box::new(SdmmcBlockQueue::::new( - dev.clone(), - self.config.name, - self.config.capacity_blocks, - self.config, - 0, - )) as _ - }) - } - - fn enable_irq(&self) { - if !self.config.irq_driven { - self.irq_enabled.store(false, Ordering::Release); - return; - } - if let Some(raw) = &self.raw { - let mut enabled = false; - raw.with_mut(|raw| { - if let Err(err) = SdioHost::enable_completion_irq(raw.host_mut()) { - warn!( - "{}: enable completion IRQ failed: {:?}", - self.config.name, err - ); - return; - } - enabled = raw.host().completion_irq_enabled(); - }); - self.irq_enabled.store(enabled, Ordering::Release); - } - } - - fn disable_irq(&self) { - if let Some(raw) = &self.raw { - raw.with_mut(|raw| { - if let Err(err) = SdioHost::disable_completion_irq(raw.host_mut()) { - warn!( - "{}: disable completion IRQ failed: {:?}", - self.config.name, err - ); - } - }); - } - self.irq_enabled.store(false, Ordering::Release); - } - - fn is_irq_enabled(&self) -> bool { - self.irq_enabled.load(Ordering::Acquire) - } - - fn irq_sources(&self) -> rdif_block::IrqSourceList { - if !self.config.irq_driven { - return Vec::new(); - } - vec![rdif_block::IrqSourceInfo::legacy( - rdif_block::IdList::from_bits(1), - )] - } - - fn take_irq_handler(&mut self, source_id: usize) -> Option> { - if !self.config.irq_driven || source_id != 0 { - return None; - } - if self.irq_handler_taken { - return None; - } - self.irq_handler_taken = true; - Some(Box::new(SdmmcBlockIrqHandler:: { - handle: self.irq_handle.clone(), - _marker: PhantomData, - })) - } -} - -struct SdmmcBlockQueue -where - H: SdmmcBlockHost, -{ - raw: SharedDriver>, - name: &'static str, - capacity_blocks: u64, - config: SdmmcBlockConfig, - id: usize, - dma: DeviceDma, - slot: H::Slot, - pending: Option, - completed: Vec, -} - -impl SdmmcBlockQueue -where - H: SdmmcBlockHost, -{ - fn new( - raw: SharedDriver>, - name: &'static str, - capacity_blocks: u64, - config: SdmmcBlockConfig, - id: usize, - ) -> Self { - let dma_mask = config.dma_mask; - Self { - raw, - name, - capacity_blocks, - config, - id, - dma: axklib::dma::device_with_mask(dma_mask), - slot: H::Slot::default(), - pending: None, - completed: Vec::new(), - } - } - - fn queue_info(&self) -> rdif_block::QueueInfo { - rdif_block::IQueue::info(self) - } - - fn submit_request_inner( - &mut self, - request: rdif_block::Request<'_>, - ) -> Result { - rdif_block::validate_request(self.queue_info(), &request)?; - self.reap_pending_request()?; - let raw = self.raw.clone(); - raw.with_mut(|raw| { - let start_block = block_addr_for_card(request.lba, raw.is_high_capacity())?; - let buffer = request - .segments - .first() - .copied() - .ok_or(rdif_block::BlkError::InvalidRequest)?; - if !buffer.len().is_multiple_of(BLOCK_SIZE) { - return Err(rdif_block::BlkError::Other("buffer is not block aligned")); - } - let ptr = NonNull::new(buffer.virt) - .ok_or(rdif_block::BlkError::Other("buffer pointer is null"))?; - let size = NonZeroUsize::new(buffer.len()) - .ok_or(rdif_block::BlkError::Other("buffer is empty"))?; - let id = match request.op { - rdif_block::RequestOp::Read => H::submit_read_request( - raw.host_mut(), - start_block, - ptr, - size, - transfer_dma(self.config.use_dma, &self.dma), - &mut self.slot, - &mut self.pending, - )?, - rdif_block::RequestOp::Write => H::submit_write_request( - raw.host_mut(), - start_block, - ptr, - size, - transfer_dma(self.config.use_dma, &self.dma), - &mut self.slot, - &mut self.pending, - )?, - rdif_block::RequestOp::Flush - | rdif_block::RequestOp::Discard - | rdif_block::RequestOp::WriteZeroes => { - return Err(rdif_block::BlkError::NotSupported); - } - }; - Ok(rdif_block::RequestId::new(usize::from(id))) - }) - } - - fn poll_request_inner( - &mut self, - request: rdif_block::RequestId, - ) -> Result { - if let Some(index) = self.completed.iter().position(|id| *id == request) { - self.completed.swap_remove(index); - return Ok(rdif_block::RequestStatus::Complete); - } - self.poll_active_request(request) - } - - fn poll_active_request( - &mut self, - request: rdif_block::RequestId, - ) -> Result { - let raw = self.raw.clone(); - match raw.with_mut(|raw| { - H::poll_block_request( - raw.host_mut(), - &mut self.pending, - BlockRequestId::new(usize::from(request)), - &mut self.slot, - ) - }) { - Ok(BlockPoll::Complete) => Ok(rdif_block::RequestStatus::Complete), - Ok(BlockPoll::Pending) => Ok(rdif_block::RequestStatus::Pending), - Ok(_) => Err(rdif_block::BlkError::Other( - "SD/MMC returned an unknown poll state", - )), - Err(err) => Err(map_dev_err_to_blk_err(err)), - } - } - - fn pending_id(&self) -> Option { - self.pending.as_ref().map(H::request_id) - } - - fn reap_pending_request(&mut self) -> Result { - let Some(active) = self.pending_id() else { - return Ok(rdif_block::RequestStatus::Complete); - }; - let id = rdif_block::RequestId::new(usize::from(active)); - match self.poll_active_request(id) { - Ok(rdif_block::RequestStatus::Complete) => { - self.completed.push(id); - Ok(rdif_block::RequestStatus::Complete) - } - Ok(rdif_block::RequestStatus::Pending) => Err(rdif_block::BlkError::Retry), - Err(err) => Err(err), - } - } -} - -// SAFETY: `SdmmcBlockQueue` owns a single pending request slot and host -// request state owns any segment access until task-side poll completes it. -unsafe impl rdif_block::IQueue for SdmmcBlockQueue -where - H: SdmmcBlockHost, -{ - fn id(&self) -> usize { - self.id - } - - fn info(&self) -> rdif_block::QueueInfo { - rdif_block::QueueInfo { - id: self.id, - device: rdif_block::DeviceInfo { - name: Some(self.name), - ..rdif_block::DeviceInfo::new(self.capacity_blocks, BLOCK_SIZE) - }, - limits: queue_limits(&self.config, self.dma.dma_mask()), - } - } - - fn submit_request( - &mut self, - request: rdif_block::Request<'_>, - ) -> Result { - self.submit_request_inner(request) - } - - fn poll_request( - &mut self, - request: rdif_block::RequestId, - ) -> Result { - self.poll_request_inner(request) - } -} - -struct SdmmcBlockIrqHandler -where - H: SdmmcBlockHost, -{ - handle: ::IrqHandle, - _marker: PhantomData, -} - -impl rdif_block::IrqHandler for SdmmcBlockIrqHandler -where - H: SdmmcBlockHost, -{ - fn handle_irq(&self) -> rdif_block::Event { - let host_event = self.handle.handle_irq(); - let mut event = rdif_block::Event::none(); - if let Some(queue_id) = block_queue_ready_from_host_event(&host_event) { - event.push_queue(queue_id); - } - event - } -} - -pub(crate) fn queue_limits(config: &SdmmcBlockConfig, dma_mask: u64) -> rdif_block::QueueLimits { - rdif_block::QueueLimits { - dma_mask, - dma_alignment: BLOCK_SIZE, - max_inflight: 1, - max_blocks_per_request: config.max_blocks_per_request, - max_segments: 1, - max_segment_size: config.max_segment_size, - supported_flags: rdif_block::RequestFlags::NONE, - supports_flush: false, - supports_discard: false, - supports_write_zeroes: false, - } -} - -pub(crate) fn device_info(config: &SdmmcBlockConfig) -> rdif_block::DeviceInfo { - rdif_block::DeviceInfo { - name: Some(config.name), - ..rdif_block::DeviceInfo::new(config.capacity_blocks, BLOCK_SIZE) - } -} - -pub(crate) fn block_addr_for_card( - block_id: u64, - high_capacity: bool, -) -> Result { - let block_id = - u32::try_from(block_id).map_err(|_| rdif_block::BlkError::InvalidBlockIndex(block_id))?; - if high_capacity { - Ok(block_id) - } else { - block_id - .checked_mul(BLOCK_SIZE as u32) - .ok_or(rdif_block::BlkError::InvalidBlockIndex(block_id as u64)) - } -} - -pub(crate) fn map_dev_err_to_blk_err(err: Error) -> rdif_block::BlkError { - match err { - Error::NoCard | Error::UnsupportedCommand | Error::CardLocked => { - rdif_block::BlkError::NotSupported - } - Error::Misaligned | Error::InvalidArgument => { - rdif_block::BlkError::Other("SD/MMC request is not block aligned") - } - _ => rdif_block::BlkError::Io, - } -} - -#[cfg(any(feature = "k230-sdhci", feature = "rockchip-sdhci"))] -impl SdmmcBlockHost for sdhci_host::Sdhci { - type Request = sdhci_host::BlockRequest; - type Slot = sdhci_host::BlockRequestSlot; - - fn submit_read_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - submit_sdhci_read_request(self, start_block, buffer, size, dma, slot, pending) - } - - fn submit_write_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - submit_sdhci_write_request(self, start_block, buffer, size, dma, slot, pending) - } - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - slot: &mut Self::Slot, - ) -> Result { - self.poll_block_request( - pending, - sdhci_host::RequestId::new(usize::from(request)), - slot, - ) - } - - fn request_id(request: &Self::Request) -> BlockRequestId { - BlockRequestId::new(usize::from(request.id())) - } -} - -#[cfg(any(feature = "k230-sdhci", feature = "rockchip-sdhci"))] -pub(crate) fn submit_sdhci_read_request( - host: &mut sdhci_host::Sdhci, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut sdhci_host::BlockRequestSlot, - pending: &mut Option, -) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match host.submit_read_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => host - .submit_read_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)?, - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) -} - -#[cfg(any(feature = "k230-sdhci", feature = "rockchip-sdhci"))] -pub(crate) fn submit_sdhci_write_request( - host: &mut sdhci_host::Sdhci, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut sdhci_host::BlockRequestSlot, - pending: &mut Option, -) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match host.submit_write_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => host - .submit_write_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)?, - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) -} - -#[cfg(any(feature = "rockchip-dwmmc", feature = "starfive-jh7110-dwmmc"))] -impl SdmmcBlockHost for dwmmc_host::DwMmc { - type Request = dwmmc_host::BlockRequest; - type Slot = dwmmc_host::BlockRequestSlot; - - fn submit_read_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match self.submit_read_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => self - .submit_read_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)?, - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) - } - - fn submit_write_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match self.submit_write_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => self - .submit_write_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)?, - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) - } - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - slot: &mut Self::Slot, - ) -> Result { - self.poll_block_request( - pending, - dwmmc_host::RequestId::new(usize::from(request)), - slot, - ) - } - - fn request_id(request: &Self::Request) -> BlockRequestId { - BlockRequestId::new(usize::from(request.id())) - } -} - -#[cfg(feature = "phytium-mci")] -impl SdmmcBlockHost for phytium_mci_host::PhytiumMci { - type Request = phytium_mci_host::BlockRequest; - type Slot = phytium_mci_host::BlockRequestSlot; - - fn submit_read_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match self.submit_read_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => { - warn!( - "phytium-mci: DMA read unavailable ({:?}); falling back to FIFO", - err - ); - self.submit_read_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)? - } - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) - } - - fn submit_write_request( - &mut self, - start_block: u32, - buffer: NonNull, - size: NonZeroUsize, - dma: Option<&DeviceDma>, - slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let request = match self.submit_write_blocks( - start_block, - buffer, - size, - dma, - transfer_mode_for_dma(dma), - slot, - ) { - Ok(request) => request, - Err(err) if dma.is_some() && can_fallback_to_fifo(err) => { - warn!( - "phytium-mci: DMA write unavailable ({:?}); falling back to FIFO", - err - ); - self.submit_write_blocks( - start_block, - buffer, - size, - None, - sdmmc_protocol::BlockTransferMode::Fifo, - slot, - ) - .map_err(map_dev_err_to_blk_err)? - } - Err(err) => return Err(map_dev_err_to_blk_err(err)), - }; - let id = request.id(); - *pending = Some(request); - Ok(BlockRequestId::new(usize::from(id))) - } - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - slot: &mut Self::Slot, - ) -> Result { - self.poll_block_request( - pending, - phytium_mci_host::RequestId::new(usize::from(request)), - slot, - ) - } - - fn request_id(request: &Self::Request) -> BlockRequestId { - BlockRequestId::new(usize::from(request.id())) - } -} - -fn transfer_mode_for_dma(dma: Option<&DeviceDma>) -> sdmmc_protocol::BlockTransferMode { - match dma { - Some(_) => sdmmc_protocol::BlockTransferMode::Dma, - None => sdmmc_protocol::BlockTransferMode::Fifo, - } -} - -fn transfer_dma(use_dma: bool, dma: &DeviceDma) -> Option<&DeviceDma> { - use_dma.then_some(dma) -} - -fn can_fallback_to_fifo(err: Error) -> bool { - matches!( - err, - Error::UnsupportedCommand | Error::InvalidArgument | Error::Misaligned - ) -} - -#[cfg(test)] -mod tests { - use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - use sdmmc_protocol::{ - CommandResponsePoll, DataCommandPoll, - cmd::Command, - sdio::{ClockSpeed, HostEvent, HostEventKind}, - }; - - use super::*; - - #[test] - fn disabled_irq_policy_does_not_advertise_sources() { - let config = SdmmcBlockConfig::dma("test-sdmmc", 8, false); - - assert_eq!(queue_limits(&config, u32::MAX as u64).max_inflight, 1); - assert_eq!(device_info(&config).name, Some("test-sdmmc")); - } - - #[test] - fn fifo_config_limits_single_block_requests() { - let config = SdmmcBlockConfig::fifo("test-sdmmc", 8, true); - let limits = queue_limits(&config, u32::MAX as u64); - - assert_eq!(limits.max_blocks_per_request, 1); - assert_eq!(limits.max_segment_size, BLOCK_SIZE); - } - - #[test] - fn irq_policy_disabled_does_not_advertise_irq_sources() { - let raw = SharedDriver::new(SdioSdmmc::new(MockHost::default())); - let device = SdmmcBlockDevice::new(raw, SdmmcBlockConfig::dma("mock-sd", 8, false)); - - assert!(rdif_block::Interface::irq_sources(&device).is_empty()); - } - - #[test] - fn enabled_irq_handler_maps_host_event_to_queue_zero() { - let raw = SharedDriver::new(SdioSdmmc::new(MockHost::default())); - let mut device = SdmmcBlockDevice::new(raw, SdmmcBlockConfig::dma("mock-sd", 8, true)); - let handler = rdif_block::Interface::take_irq_handler(&mut device, 0).unwrap(); - - let event = handler.handle_irq(); - - assert!(event.queues.contains(0)); - assert!(!event.is_empty()); - } - - #[test] - fn poll_request_only_completes_matching_request_id() { - let raw = SharedDriver::new(SdioSdmmc::new(MockHost::default())); - let mut queue = SdmmcBlockQueue:: { - raw, - name: "mock-sd", - capacity_blocks: 8, - config: SdmmcBlockConfig::dma("mock-sd", 8, false), - id: 0, - dma: axklib::dma::device_with_mask(u32::MAX as u64), - slot: MockSlot, - pending: Some(MockRequest { - id: BlockRequestId::new(7), - }), - completed: Vec::new(), - }; - - assert_eq!( - queue.poll_request_inner(rdif_block::RequestId::new(8)), - Ok(rdif_block::RequestStatus::Pending) - ); - assert_eq!( - queue.poll_request_inner(rdif_block::RequestId::new(7)), - Ok(rdif_block::RequestStatus::Complete) - ); - assert!(queue.pending.is_none()); - } - - #[derive(Clone, Default)] - struct MockIrqHandle; - - impl SdioIrqHandle for MockIrqHandle { - type Event = MockEvent; - - fn handle_irq(&self) -> Self::Event { - MockEvent(HostEventKind::TransferComplete) - } - } - - #[derive(Clone, Copy, Default)] - struct MockEvent(HostEventKind); - - impl HostEvent for MockEvent { - fn kind(&self) -> HostEventKind { - self.0 - } - } - - #[derive(Default)] - struct MockHost { - irq_enabled: AtomicBool, - next_id: AtomicUsize, - } - - #[derive(Default)] - struct MockSlot; - - struct MockRequest { - id: BlockRequestId, - } - - impl SdioHost for MockHost { - type Event = MockEvent; - type DataRequest<'a> = (); - - fn submit_command(&mut self, _cmd: &Command) -> Result<(), Error> { - Err(Error::UnsupportedCommand) - } - - fn poll_command_response(&mut self) -> Result { - Ok(CommandResponsePoll::Pending) - } - - fn submit_read_data<'a>( - &mut self, - _cmd: &Command, - _buf: &'a mut [u8], - _block_size: u32, - _block_count: u32, - ) -> Result, Error> { - Err(Error::UnsupportedCommand) - } - - fn submit_write_data<'a>( - &mut self, - _cmd: &Command, - _buf: &'a [u8], - _block_size: u32, - _block_count: u32, - ) -> Result, Error> { - Err(Error::UnsupportedCommand) - } - - fn poll_data_request<'a>( - &mut self, - _request: &mut Self::DataRequest<'a>, - ) -> Result { - Err(Error::UnsupportedCommand) - } - - fn set_bus_width(&mut self, _width: sdmmc_protocol::sdio::BusWidth) -> Result<(), Error> { - Ok(()) - } - - fn set_clock(&mut self, _speed: ClockSpeed) -> Result<(), Error> { - Ok(()) - } - - fn enable_completion_irq(&mut self) -> Result<(), Error> { - self.irq_enabled.store(true, Ordering::Release); - Ok(()) - } - - fn disable_completion_irq(&mut self) -> Result<(), Error> { - self.irq_enabled.store(false, Ordering::Release); - Ok(()) - } - } - - impl SdioIrqHost for MockHost { - type IrqHandle = MockIrqHandle; - - fn irq_handle(&self) -> Self::IrqHandle { - MockIrqHandle - } - - fn completion_irq_enabled(&self) -> bool { - self.irq_enabled.load(Ordering::Acquire) - } - } - - impl SdmmcBlockHost for MockHost { - type Request = MockRequest; - type Slot = MockSlot; - - fn submit_read_request( - &mut self, - _start_block: u32, - _buffer: NonNull, - _size: NonZeroUsize, - _dma: Option<&DeviceDma>, - _slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - self.submit_mock_request(pending) - } - - fn submit_write_request( - &mut self, - _start_block: u32, - _buffer: NonNull, - _size: NonZeroUsize, - _dma: Option<&DeviceDma>, - _slot: &mut Self::Slot, - pending: &mut Option, - ) -> Result { - self.submit_mock_request(pending) - } - - fn poll_block_request( - &mut self, - pending: &mut Option, - request: BlockRequestId, - _slot: &mut Self::Slot, - ) -> Result { - match pending.as_ref() { - Some(active) if active.id == request => { - *pending = None; - Ok(BlockPoll::Complete) - } - Some(_) => Ok(BlockPoll::Pending), - None => Ok(BlockPoll::Complete), - } - } - - fn request_id(request: &Self::Request) -> BlockRequestId { - request.id - } - } - - impl MockHost { - fn submit_mock_request( - &self, - pending: &mut Option, - ) -> Result { - if pending.is_some() { - return Err(rdif_block::BlkError::Retry); - } - let id = BlockRequestId::new(self.next_id.fetch_add(1, Ordering::Relaxed)); - *pending = Some(MockRequest { id }); - Ok(id) - } - } -} diff --git a/drivers/ax-driver/src/block/starfive_mmc.rs b/drivers/ax-driver/src/block/starfive_mmc.rs index 3f2e99fe32..aaffe6845e 100644 --- a/drivers/ax-driver/src/block/starfive_mmc.rs +++ b/drivers/ax-driver/src/block/starfive_mmc.rs @@ -1,7 +1,7 @@ use alloc::format; use core::time::Duration; -use dwmmc_host::DwMmc; +use dwmmc_host::{DwMmc, rdif as dwmmc_rdif}; use log::{info, warn}; use rdrive::{ probe::OnProbeError, @@ -10,21 +10,15 @@ use rdrive::{ use sdmmc_protocol::{ Error, OperationPoll, error::Phase, - sdio::{CardInfo, SdioInitScratch, SdioSdmmc}, + sdio::{CardInfo, SdioHost2Adapter, SdioInitScratch, SdioSdmmc}, }; -use crate::{ - block::{ - ProbeFdtBlock, SharedDriver, - sdmmc::{SdmmcBlockConfig, SdmmcBlockDevice}, - }, - mmio::iomap, -}; +use crate::{block::ProbeFdtBlock, mmio::iomap}; const STARFIVE_JH7110_MMC1_BASE: u64 = 0x1602_0000; const DWMMC_STABLE_REFERENCE_CLOCK: u32 = 50_000_000; -type StarFiveDwMmc = SdioSdmmc; +type StarFiveDwMmc = SdioSdmmc>; crate::model_register!( name: "StarFive JH7110 MMC", @@ -77,7 +71,7 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { .map_err(|err| init_error(address, mmio_size, err))?; info!("starfive-jh7110-dwmmc: initialize card"); - let mut sd = SdioSdmmc::new(host); + let mut sd = SdioSdmmc::new_host2(host); sd.set_sd_speed_selection_enabled(false); let card_info = poll_card_init(&mut sd).map_err(|err| { warn!("starfive-jh7110-dwmmc: card init failed: {:?}", err); @@ -95,10 +89,9 @@ fn probe(probe: ProbeFdt<'_>) -> Result<(), OnProbeError> { card_info.ext_csd.is_some() ); - let raw = SharedDriver::new(sd); - let dev = SdmmcBlockDevice::new( - raw, - SdmmcBlockConfig::fifo( + let dev = dwmmc_rdif::device( + sd, + dwmmc_rdif::fifo_config( "starfive-jh7110-mmc", card_info.capacity_blocks.unwrap_or(0), false, diff --git a/drivers/ax-driver/src/virtio/block.rs b/drivers/ax-driver/src/virtio/block.rs index 4ba0959a21..5c8e816dee 100644 --- a/drivers/ax-driver/src/virtio/block.rs +++ b/drivers/ax-driver/src/virtio/block.rs @@ -110,6 +110,7 @@ impl rdif_block::Interface for BlockDevice { fn queue_limits(&self) -> rdif_block::QueueLimits { rdif_block::QueueLimits { + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_mask: u64::MAX, dma_alignment: 0x1000, max_inflight: 1, @@ -173,6 +174,7 @@ unsafe impl rdif_block::IQueue for BlockQueue { ..rdif_block::DeviceInfo::new(blocks, SECTOR_SIZE) }, limits: rdif_block::QueueLimits { + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_mask: u64::MAX, dma_alignment: 0x1000, max_inflight: 1, diff --git a/drivers/blk/dwmmc-host/Cargo.toml b/drivers/blk/dwmmc-host/Cargo.toml index 41b4156893..967556b01e 100644 --- a/drivers/blk/dwmmc-host/Cargo.toml +++ b/drivers/blk/dwmmc-host/Cargo.toml @@ -10,13 +10,15 @@ keywords = ["sd", "mmc", "dwmmc", "embedded", "no_std"] categories = ["embedded", "no-std", "hardware-support"] [dependencies] -sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio"] } +sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio", "rdif"] } +sdio-host2.workspace = true embedded-hal = "1" bitfield-struct = "0.11" volatile = { version = "0.6", features = ["derive"] } log.workspace = true dma-api.workspace = true mmio-api.workspace = true +rdif-block.workspace = true [features] default = [] diff --git a/drivers/blk/dwmmc-host/README.md b/drivers/blk/dwmmc-host/README.md index 31ca104982..789f5d072c 100644 --- a/drivers/blk/dwmmc-host/README.md +++ b/drivers/blk/dwmmc-host/README.md @@ -4,19 +4,19 @@ backend for [`sdmmc-protocol`](../sdmmc-protocol). This crate plugs the IP block known as `DWC_mobile_storage` / `dw_mshc` / -`dw_mmc` (Linux) into the `SdioHost` trait so -`sdmmc_protocol::sdio::SdioSdmmc` can drive real hardware. The same core +`dw_mmc` (Linux) into the physical `sdio_host2::SdioHost` trait so +`sdmmc_protocol::sdio::SdioSdmmc::new_host2` can drive real hardware. The same core appears in Rockchip RK33xx/RK35xx, Allwinner A-series, StarFive JH7110, and a long tail of mid-range SoCs. -This crate implements `sdmmc_protocol::sdio::SdioHost` for the controller while +This crate implements `sdio_host2::SdioHost` for the controller while leaving MMIO mapping, SoC clocks, resets, pinmux, power rails, IRQ routing, and DMA cache policy to platform glue. ## Status - Compiles as a `no_std` controller backend. -- Intended for use through `sdmmc_protocol::sdio::SdioSdmmc`. +- Intended for use through `sdmmc_protocol::sdio::SdioSdmmc::new_host2`. - Board-specific clock, power, pinmux, and tuning policy must be supplied by the caller. - Real hardware bring-up still depends on the surrounding SoC integration. @@ -58,9 +58,10 @@ use dwmmc_host::DwMmc; let mmio = NonNull::new(0xFE2B_0000 as *mut u8).unwrap(); let mut host = unsafe { DwMmc::new(mmio) }; host.set_reference_clock(50_000_000); -host.reset_and_init().expect("controller reset"); +// Optional DMA capability can be installed here before the protocol layer owns +// the host. -let mut card = SdioSdmmc::new(host); +let mut card = SdioSdmmc::new_host2(host); let mut scratch = SdioInitScratch::new(); let mut request = card.submit_init(&mut scratch)?; while let OperationPoll::Pending = card.poll_init_request(&mut request)? { @@ -83,15 +84,22 @@ the driver. to 400 kHz for ID mode. 3. Pass that rate to `DwMmc::set_reference_clock` so the divider programmed by `set_clock` lands on the right frequency. -4. `host.reset_and_init()?` — clears the controller / FIFO / DMA - state and arms a 400 kHz ID-mode clock. -5. Build `SdioSdmmc::new(host)`, submit initialization with - `submit_init`, and drive it with `poll_init_request`. The - protocol layer will ramp the clock up via `set_clock`; platform/runtime - code chooses whether pending work spins, yields, or waits for an IRQ. +4. Install optional capabilities such as `DwMmc::set_dma` before handing the + host to the protocol layer. +5. Build `SdioSdmmc::new_host2(host)`, submit initialization with + `submit_init`, and drive it with `poll_init_request`. The protocol layer + starts with native `sdio-host2` bus operations for `ResetAll`, `PowerOn`, + initial voltage, 1-bit bus width, and 400 kHz identification clock before + issuing SD/MMC commands, then ramps the clock up via later bus ops. + Platform/runtime code chooses whether pending work spins, yields, or waits + for an IRQ. 6. Add board-specific tuning before relying on SDR50, SDR104, DDR50, or HS200 modes. +The lower-level blocking helper `DwMmc::reset_and_init` remains useful for +diagnostics, but normal card initialization should let `SdioSdmmc::new_host2` +drive reset, power, and clock setup through submit/poll bus operations. + ### FIFO offset The data FIFO sits at a fixed offset that varies by IP revision / diff --git a/drivers/blk/dwmmc-host/src/command.rs b/drivers/blk/dwmmc-host/src/command.rs index ffe90c2d9d..9a81ab5ccb 100644 --- a/drivers/blk/dwmmc-host/src/command.rs +++ b/drivers/blk/dwmmc-host/src/command.rs @@ -7,7 +7,7 @@ use sdmmc_protocol::{ CommandPoll, CommandResponsePoll, cmd::{Command as ProtoCmd, DataDirection}, - error::{Error, Phase}, + error::{Error, ErrorContext, Phase}, response::{ IfCondResponse, OcrResponse, R1Response, RcaResponse, Response, ResponseType, SdioOcrResponse, SdioRwResponse, @@ -25,12 +25,15 @@ pub(crate) enum CommandState { WaitingInhibit { cmd: ProtoCmd, data: Option, + polls: u32, }, WaitingStart { cmd: ProtoCmd, + polls: u32, }, Issued { cmd: ProtoCmd, + polls: u32, }, Complete { response: Response, @@ -62,7 +65,12 @@ impl DwMmc { return Err(Error::UnsupportedCommand); } let data = self.pending_data.take(); - self.command_state = CommandState::WaitingInhibit { cmd: *cmd, data }; + self.prepare_irq_for_request(); + self.command_state = CommandState::WaitingInhibit { + cmd: *cmd, + data, + polls: 0, + }; if let Err(err) = self.poll_command() { self.command_state = CommandState::Idle; return Err(err); @@ -72,18 +80,39 @@ impl DwMmc { pub fn poll_command(&mut self) -> Result { match self.command_state { - CommandState::WaitingInhibit { cmd, data } => { + CommandState::WaitingInhibit { cmd, data, polls } => { if !self.command_can_issue(data.is_some()) { + if polls >= COMMAND_WAIT_POLLS { + let err = + Error::Timeout(ErrorContext::for_cmd(Phase::CommandSend, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::WaitingInhibit { + cmd, + data, + polls: polls + 1, + }; return Ok(CommandPoll::Pending); } self.program_command(&cmd, data); return Ok(CommandPoll::Pending); } - CommandState::WaitingStart { cmd } => { + CommandState::WaitingStart { cmd, polls } => { if self.regs.cmd().read().start_cmd() { + if polls >= COMMAND_WAIT_POLLS { + let err = + Error::Timeout(ErrorContext::for_cmd(Phase::CommandSend, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::WaitingStart { + cmd, + polls: polls + 1, + }; return Ok(CommandPoll::Pending); } - self.command_state = CommandState::Issued { cmd }; + self.command_state = CommandState::Issued { cmd, polls: 0 }; return Ok(CommandPoll::Pending); } CommandState::Issued { .. } => {} @@ -92,18 +121,18 @@ impl DwMmc { CommandState::Idle => return Err(Error::InvalidArgument), } - let CommandState::Issued { cmd } = self.command_state else { + let CommandState::Issued { cmd, polls } = self.command_state else { unreachable!(); }; let raw_status = self.take_command_irq_status(); let status = crate::regs::RIntSts::from_bits(raw_status); if status.error() { - let err = self.translate_int_error(status, Phase::ResponseWait, cmd.cmd); + let err = self.translate_int_error(status, Phase::ResponseWait, cmd.index); self.command_state = CommandState::Failed { error: err }; return Err(err); } if status.command_done() { - let response = match decode_response(self, cmd.resp_type) { + let response = match decode_response(self, cmd.response) { Ok(r) => r, Err(err) => { // Park the FSM in Failed before propagating: bare `?` would @@ -117,6 +146,15 @@ impl DwMmc { self.command_state = CommandState::Complete { response }; return Ok(CommandPoll::Complete); } + if polls >= COMMAND_WAIT_POLLS { + let err = Error::Timeout(ErrorContext::for_cmd(Phase::ResponseWait, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::Issued { + cmd, + polls: polls + 1, + }; Ok(CommandPoll::Pending) } @@ -124,10 +162,14 @@ impl DwMmc { match self.command_state { CommandState::Complete { response } => { self.command_state = CommandState::Idle; + if self.data_cmd_index == 0 { + self.irq.state.end_request(); + } Ok(response) } CommandState::Failed { error } => { self.command_state = CommandState::Idle; + self.irq.state.end_request(); Err(error) } CommandState::Idle @@ -145,19 +187,28 @@ impl DwMmc { fn program_command(&mut self, cmd: &ProtoCmd, data: Option) { if data.is_some() { - self.data_cmd_index = cmd.cmd; + self.data_cmd_index = cmd.index; } self.clear_command_int_status(); let data_dir = data.map(|d| { self.program_data_phase(d.block_size, d.block_count); d.direction }); - self.regs.cmdarg().write(cmd.arg); + self.regs.cmdarg().write(cmd.argument); self.regs.cmd().write(encode_command(cmd, data_dir)); - self.command_state = CommandState::WaitingStart { cmd: *cmd }; + self.command_state = CommandState::WaitingStart { + cmd: *cmd, + polls: 0, + }; } fn take_command_irq_status(&mut self) -> u32 { + if self.completion_irq_enabled() { + return self + .irq + .state + .take(crate::DWMMC_INT_COMMAND_DONE | crate::DWMMC_INT_ERROR_MASK); + } let raw_status = self.regs.rintsts().read().into_bits(); let consume = raw_status & (crate::DWMMC_INT_COMMAND_DONE | crate::DWMMC_INT_ERROR_MASK); if consume != 0 { @@ -165,9 +216,7 @@ impl DwMmc { .rintsts() .write(crate::regs::RIntSts::from_bits(consume)); } - self.irq_state - .take(crate::DWMMC_INT_COMMAND_DONE | crate::DWMMC_INT_ERROR_MASK) - | raw_status + raw_status } fn clear_command_int_status(&mut self) { @@ -178,11 +227,41 @@ impl DwMmc { .rintsts() .write(crate::regs::RIntSts::from_bits(raw_status)); } - self.irq_state + self.irq + .state .clear(crate::DWMMC_INT_COMMAND_DONE | crate::DWMMC_INT_ERROR_MASK); } + + fn prepare_irq_for_request(&mut self) { + self.clear_all_int_status(); + self.irq.state.begin_request(); + } + + pub(crate) fn abort_command(&mut self) -> Result<(), Error> { + self.clear_command_int_status(); + for _ in 0..COMMAND_WAIT_POLLS { + if !self.regs.cmd().read().start_cmd() { + self.clear_all_int_status(); + self.reset_and_init_preserving_irq()?; + self.pending_data = None; + self.data_blocks_remaining = 0; + self.data_cmd_index = 0; + self.command_state = CommandState::Idle; + return Ok(()); + } + core::hint::spin_loop(); + } + self.reset_and_init_preserving_irq()?; + self.pending_data = None; + self.data_blocks_remaining = 0; + self.data_cmd_index = 0; + self.command_state = CommandState::Idle; + Ok(()) + } } +const COMMAND_WAIT_POLLS: u32 = 1_000_000; + /// Build the CMD register value for a single command. /// /// `data_dir` is `Some` when the command carries a data phase (the @@ -196,9 +275,9 @@ fn encode_command(cmd: &ProtoCmd, data_dir: Option) -> Cmd { .with_start_cmd(true) .with_use_hold_reg(true) .with_wait_prvdata_complete(true) - .with_cmd_index(cmd.cmd & 0x3F); + .with_cmd_index(cmd.index & 0x3F); - match cmd.resp_type { + match cmd.response { ResponseType::None => { // No response_expect; no CRC check. } @@ -230,7 +309,7 @@ fn encode_command(cmd: &ProtoCmd, data_dir: Option) -> Cmd { _ => {} } - if cmd.cmd == 0 { + if cmd.index == 0 { // Power-up cards need 80 init clocks before CMD0. c = c.with_send_initialization(true); } @@ -241,7 +320,7 @@ fn encode_command(cmd: &ProtoCmd, data_dir: Option) -> Cmd { // some indices are overloaded (CMD6 = ACMD6 SET_BUS_WIDTH no-data / // SWITCH_FUNC read; CMD8 = SEND_IF_COND no-data on SD / // SEND_EXT_CSD read on MMC). We trust that signal here rather than - // inferring from `cmd.cmd`. + // inferring from `cmd.index`. if matches!(dir, DataDirection::Write) { c = c.with_read_write(true); } diff --git a/drivers/blk/dwmmc-host/src/dma.rs b/drivers/blk/dwmmc-host/src/dma.rs index d7077bdaab..5f44b65be2 100644 --- a/drivers/blk/dwmmc-host/src/dma.rs +++ b/drivers/blk/dwmmc-host/src/dma.rs @@ -1,6 +1,9 @@ +use alloc::boxed::Box; use core::{num::NonZeroUsize, ptr::NonNull}; -use dma_api::{CoherentArray, DeviceDma, DmaDirection, StreamingMap}; +use dma_api::{ + CoherentArray, CompletedDma, CpuDmaBuffer, DeviceDma, DmaDirection, InFlightDma, PreparedDma, +}; use log::warn; use sdmmc_protocol::{ block::{ @@ -38,9 +41,14 @@ pub type RequestId = BlockRequestId; pub struct BlockRequestSlot { next: usize, state: BlockTransferState, + completed_dma: Option, } impl BlockRequestSlot { + pub fn take_completed_dma(&mut self) -> Option { + self.completed_dma.take() + } + pub fn start( &mut self, mode: BlockTransferMode, @@ -60,10 +68,19 @@ impl BlockRequestSlot { } pub fn complete(&mut self, id: RequestId) -> Result<(), Error> { + self.complete_with_dma(id, None) + } + + fn complete_with_dma( + &mut self, + id: RequestId, + completed_dma: Option, + ) -> Result<(), Error> { if self.state.id() != Some(id) { return Err(Error::InvalidArgument); } self.state = BlockTransferState::Idle; + self.completed_dma = completed_dma; Ok(()) } @@ -76,6 +93,24 @@ pub struct BlockRequest { inner: BlockRequestKind, } +pub struct PreparedDmaSubmitError { + pub error: Error, + buffer: Box, +} + +impl PreparedDmaSubmitError { + fn new(error: Error, buffer: PreparedDma) -> Self { + Self { + error, + buffer: Box::new(buffer), + } + } + + pub fn into_buffer(self) -> PreparedDma { + *self.buffer + } +} + // `BlockRequest` owns the DMA mappings and descriptor buffer for one // submitted transfer. Moving that ownership to another queue thread does not // grant shared access to the mapped memory; completion still requires a @@ -109,7 +144,7 @@ enum BlockRequestKind { }, Read { id: RequestId, - map: StreamingMap, + buffer: DmaRequestBuffer, _desc: CoherentArray, cmd_index: u8, phase: Phase, @@ -119,7 +154,7 @@ enum BlockRequestKind { }, Write { id: RequestId, - _map: StreamingMap, + buffer: DmaRequestBuffer, _desc: CoherentArray, cmd_index: u8, phase: Phase, @@ -129,6 +164,54 @@ enum BlockRequestKind { }, } +enum DmaRequestBuffer { + Bounce { + buffer: InFlightDma, + readback: Option<(NonNull, usize)>, + }, + Owned(InFlightDma), +} + +impl DmaRequestBuffer { + fn complete(self, read: bool) -> Option { + self.finish(read, true) + } + + fn abort(self, read: bool, quiesced: bool) -> Option { + self.finish(read, quiesced) + } + + fn finish(self, read: bool, quiesced: bool) -> Option { + match self { + Self::Bounce { buffer, readback } => { + if !quiesced { + let _quarantined = buffer.quarantine(); + return None; + } + if read { + let completed = unsafe { buffer.complete_after_quiesce() }; + if let Some((dst, len)) = readback { + completed.copy_from_device_to_slice(unsafe { + core::slice::from_raw_parts_mut(dst.as_ptr(), len) + }); + } + None + } else { + drop(unsafe { buffer.complete_after_quiesce() }); + None + } + } + Self::Owned(in_flight) => { + if !quiesced { + let _quarantined = in_flight.quarantine(); + return None; + } + Some(unsafe { in_flight.complete_after_quiesce() }) + } + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum BlockRequestStage { Command, @@ -223,6 +306,7 @@ impl DwMmc { mode: BlockTransferMode, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let id = slot.start(mode, BlockTransferDirection::Read)?; let result = match mode { BlockTransferMode::Dma => { @@ -254,6 +338,7 @@ impl DwMmc { mode: BlockTransferMode, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let id = slot.start(mode, BlockTransferDirection::Write)?; let result = match mode { BlockTransferMode::Dma => { @@ -273,6 +358,52 @@ impl DwMmc { } } + pub fn submit_prepared_read_blocks( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + slot: &mut BlockRequestSlot, + ) -> Result { + if let Err(err) = self.check_not_poisoned() { + return Err(PreparedDmaSubmitError::new(err, buffer)); + } + let id = match slot.start(BlockTransferMode::Dma, BlockTransferDirection::Read) { + Ok(id) => id, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + match self.build_prepared_dma_read_request(start_block, buffer, dma, id) { + Ok(request) => Ok(request), + Err(err) => { + let _ = slot.complete(id); + Err(err) + } + } + } + + pub fn submit_prepared_write_blocks( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + slot: &mut BlockRequestSlot, + ) -> Result { + if let Err(err) = self.check_not_poisoned() { + return Err(PreparedDmaSubmitError::new(err, buffer)); + } + let id = match slot.start(BlockTransferMode::Dma, BlockTransferDirection::Write) { + Ok(id) => id, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + match self.build_prepared_dma_write_request(start_block, buffer, dma, id) { + Ok(request) => Ok(request), + Err(err) => { + let _ = slot.complete(id); + Err(err) + } + } + } + /// Poll a previously submitted block request. pub fn poll_block_request( &mut self, @@ -355,7 +486,7 @@ impl DwMmc { // Future CommandPoll variants: best-effort, treat as still pending. Ok(_) => return Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); return Err(err); } } @@ -371,12 +502,21 @@ impl DwMmc { // Future BlockPoll variants: best-effort, treat as still pending. Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); Err(err) } } } + pub fn abort_block_request_response( + &mut self, + request: &mut Option, + id: RequestId, + slot: &mut BlockRequestSlot, + ) -> Result<(), Error> { + self.abort_block_request(request, id, slot, Phase::DataRead) + } + fn build_dma_read_request( &mut self, start_block: u32, @@ -386,13 +526,10 @@ impl DwMmc { id: RequestId, ) -> Result { let block_count = dma_read_block_count(size)?; - let map = dma - .map_streaming_slice_for_device( - unsafe { core::slice::from_raw_parts_mut(buffer.as_ptr(), size.get()) }, - BLOCK_SIZE, - DmaDirection::FromDevice, - ) + let backing = CpuDmaBuffer::new_zero(dma, size, BLOCK_SIZE, DmaDirection::FromDevice) .map_err(|err| map_dma_error(err, Phase::DataRead))?; + let dma_addr = backing.dma_addr().as_u64(); + let in_flight = unsafe { backing.prepare_for_device().into_in_flight() }; let mut desc = dma .coherent_array_zero_with_align::(block_count as usize, IDMAC_DESC_ALIGN) .map_err(|err| map_dma_error(err, Phase::DataRead))?; @@ -401,13 +538,16 @@ impl DwMmc { } else { cmd18(start_block) }; - self.submit_idmac_transfer_mapped(&cmd, block_count, map.dma_addr().as_u64(), &mut desc)?; + self.submit_idmac_transfer_mapped(&cmd, block_count, dma_addr, &mut desc)?; Ok(BlockRequest { inner: BlockRequestKind::Read { id, - map, + buffer: DmaRequestBuffer::Bounce { + buffer: in_flight, + readback: Some((buffer, size.get())), + }, _desc: desc, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase: Phase::DataRead, stage: BlockRequestStage::Command, stop_after_complete: block_count > 1, @@ -425,13 +565,13 @@ impl DwMmc { id: RequestId, ) -> Result { let block_count = dma_write_block_count(size)?; - let map = dma - .map_streaming_slice_for_device( - unsafe { core::slice::from_raw_parts_mut(buffer.as_ptr(), size.get()) }, - BLOCK_SIZE, - DmaDirection::ToDevice, - ) + let mut backing = CpuDmaBuffer::new_zero(dma, size, BLOCK_SIZE, DmaDirection::ToDevice) .map_err(|err| map_dma_error(err, Phase::DataWrite))?; + backing.copy_to_device_from_slice(unsafe { + core::slice::from_raw_parts(buffer.as_ptr(), size.get()) + }); + let dma_addr = backing.dma_addr().as_u64(); + let in_flight = unsafe { backing.prepare_for_device().into_in_flight() }; let mut desc = dma .coherent_array_zero_with_align::(block_count as usize, IDMAC_DESC_ALIGN) .map_err(|err| map_dma_error(err, Phase::DataWrite))?; @@ -440,13 +580,124 @@ impl DwMmc { } else { cmd25(start_block) }; - self.submit_idmac_transfer_mapped(&cmd, block_count, map.dma_addr().as_u64(), &mut desc)?; + self.submit_idmac_transfer_mapped(&cmd, block_count, dma_addr, &mut desc)?; Ok(BlockRequest { inner: BlockRequestKind::Write { id, - _map: map, + buffer: DmaRequestBuffer::Bounce { + buffer: in_flight, + readback: None, + }, _desc: desc, - cmd_index: cmd.cmd, + cmd_index: cmd.index, + phase: Phase::DataWrite, + stage: BlockRequestStage::Command, + stop_after_complete: block_count > 1, + response: None, + }, + }) + } + + fn build_prepared_dma_read_request( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + id: RequestId, + ) -> Result { + if buffer.direction() != DmaDirection::FromDevice || buffer.domain_id() != dma.domain_id() { + return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)); + } + let block_count = match dma_read_block_count(buffer.len()) { + Ok(block_count) => block_count, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + let mut desc = match dma + .coherent_array_zero_with_align::(block_count as usize, IDMAC_DESC_ALIGN) + { + Ok(desc) => desc, + Err(err) => { + return Err(PreparedDmaSubmitError::new( + map_dma_error(err, Phase::DataRead), + buffer, + )); + } + }; + let cmd = if block_count == 1 { + cmd17(start_block) + } else { + cmd18(start_block) + }; + match self.submit_idmac_transfer_mapped( + &cmd, + block_count, + buffer.dma_addr().as_u64(), + &mut desc, + ) { + Ok(()) => {} + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + } + let buffer = unsafe { buffer.into_in_flight() }; + Ok(BlockRequest { + inner: BlockRequestKind::Read { + id, + buffer: DmaRequestBuffer::Owned(buffer), + _desc: desc, + cmd_index: cmd.index, + phase: Phase::DataRead, + stage: BlockRequestStage::Command, + stop_after_complete: block_count > 1, + response: None, + }, + }) + } + + fn build_prepared_dma_write_request( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + id: RequestId, + ) -> Result { + if buffer.direction() != DmaDirection::ToDevice || buffer.domain_id() != dma.domain_id() { + return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)); + } + let block_count = match dma_write_block_count(buffer.len()) { + Ok(block_count) => block_count, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + let mut desc = match dma + .coherent_array_zero_with_align::(block_count as usize, IDMAC_DESC_ALIGN) + { + Ok(desc) => desc, + Err(err) => { + return Err(PreparedDmaSubmitError::new( + map_dma_error(err, Phase::DataWrite), + buffer, + )); + } + }; + let cmd = if block_count == 1 { + cmd24(start_block) + } else { + cmd25(start_block) + }; + match self.submit_idmac_transfer_mapped( + &cmd, + block_count, + buffer.dma_addr().as_u64(), + &mut desc, + ) { + Ok(()) => {} + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + } + let buffer = unsafe { buffer.into_in_flight() }; + Ok(BlockRequest { + inner: BlockRequestKind::Write { + id, + buffer: DmaRequestBuffer::Owned(buffer), + _desc: desc, + cmd_index: cmd.index, phase: Phase::DataWrite, stage: BlockRequestStage::Command, stop_after_complete: block_count > 1, @@ -516,6 +767,7 @@ impl DwMmc { direction: DataDirection, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let transfer_direction = match direction { DataDirection::Read => BlockTransferDirection::Read, DataDirection::Write => BlockTransferDirection::Write, @@ -583,7 +835,7 @@ impl DwMmc { len, block_size: block_size_usize, offset: 0, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase, stage: BlockRequestStage::Command, stop_after_complete, @@ -595,7 +847,7 @@ impl DwMmc { len, block_size: block_size_usize, offset: 0, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase, stage: BlockRequestStage::Command, stop_after_complete, @@ -618,13 +870,12 @@ impl DwMmc { if block_count == 0 { return Err(Error::InvalidArgument); } - let direction = cmd.data_direction(); - let phase = match direction { - DataDirection::Read => Phase::DataRead, - DataDirection::Write => Phase::DataWrite, - DataDirection::None => return Err(Error::InvalidArgument), + let (direction, phase) = match cmd.data_direction() { + Some(sdio_host2::DataDirection::Read) => (DataDirection::Read, Phase::DataRead), + Some(sdio_host2::DataDirection::Write) => (DataDirection::Write, Phase::DataWrite), + None => return Err(Error::InvalidArgument), // Future DataDirection variants are not supported by this engine. - _ => return Err(Error::InvalidArgument), + Some(_) => return Err(Error::InvalidArgument), }; let byte_count = block_count .checked_mul(BLOCK_SIZE as u32) @@ -665,7 +916,7 @@ impl DwMmc { }); self.clear_all_int_status(); - self.irq_state.clear(u32::MAX); + self.irq.state.clear(u32::MAX); self.program_data_phase(BLOCK_SIZE as u32, block_count); self.reset_dma_for_phase(phase)?; @@ -688,21 +939,42 @@ impl DwMmc { Ok(()) => Ok(()), Err(err) => { self.disable_idmac(); - self.recover_after_idmac_error(phase); + let _ = self.recover_after_idmac_error(phase); self.clear_all_int_status(); Err(err) } } } - fn finish_block_request(&mut self, request: BlockRequest) -> Result<(), Error> { - match request.inner { + fn finish_block_request( + &mut self, + request: BlockRequest, + ) -> Result, Error> { + self.finish_block_request_with_quiesce(request, true) + } + + fn finish_block_request_with_quiesce( + &mut self, + request: BlockRequest, + quiesced: bool, + ) -> Result, Error> { + if !quiesced { + self.poison_dma(); + core::mem::forget(request); + self.pending_data = None; + self.data_blocks_remaining = 0; + self.data_cmd_index = 0; + self.irq.state.end_request(); + return Ok(None); + } + let completed_dma = match request.inner { BlockRequestKind::FifoRead { .. } | BlockRequestKind::FifoWrite { .. } => { self.pending_data = None; self.data_blocks_remaining = 0; self.data_cmd_index = 0; + None } - BlockRequestKind::Read { stage, .. } => { + BlockRequestKind::Read { stage, buffer, .. } => { if stage == BlockRequestStage::Command { let _ = self.take_command_response(); } @@ -711,8 +983,13 @@ impl DwMmc { self.pending_data = None; self.data_blocks_remaining = 0; self.data_cmd_index = 0; + if quiesced { + buffer.complete(true) + } else { + buffer.abort(true, false) + } } - BlockRequestKind::Write { stage, .. } => { + BlockRequestKind::Write { stage, buffer, .. } => { if stage == BlockRequestStage::Command { let _ = self.take_command_response(); } @@ -721,9 +998,15 @@ impl DwMmc { self.pending_data = None; self.data_blocks_remaining = 0; self.data_cmd_index = 0; + if quiesced { + buffer.complete(false) + } else { + buffer.abort(false, false) + } } - } - Ok(()) + }; + self.irq.state.end_request(); + Ok(completed_dma) } fn finish_dma_data( @@ -737,12 +1020,10 @@ impl DwMmc { }; let stop_after_complete = match &mut active.inner { BlockRequestKind::Read { - map, stage, stop_after_complete, .. } => { - map.complete_for_cpu_all(); *stage = BlockRequestStage::Stop; *stop_after_complete } @@ -766,8 +1047,8 @@ impl DwMmc { let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active)?; - slot.complete(id)?; + let completed_dma = self.finish_block_request(active)?; + slot.complete_with_dma(id, completed_dma)?; Ok(DataCommandPoll::Complete(response)) } @@ -784,14 +1065,14 @@ impl DwMmc { let _ = self.take_command_response()?; let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active)?; - slot.complete(id)?; + let completed_dma = self.finish_block_request(active)?; + slot.complete_with_dma(id, completed_dma)?; Ok(DataCommandPoll::Complete(response)) } // Future CommandPoll variants: best-effort, treat as still pending. Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); Err(err) } } @@ -843,7 +1124,7 @@ impl DwMmc { // Future CommandPoll variants: best-effort, treat as still pending. Ok(_) => return Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); return Err(err); } } @@ -864,7 +1145,7 @@ impl DwMmc { // Future BlockPoll variants: best-effort, treat as still pending. Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); Err(err) } } @@ -930,7 +1211,8 @@ impl DwMmc { let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active)?; + let completed_dma = self.finish_block_request(active)?; + drop(completed_dma); self.pending_data = None; self.data_blocks_remaining = 0; self.data_cmd_index = 0; @@ -944,12 +1226,22 @@ impl DwMmc { id: RequestId, slot: &mut BlockRequestSlot, phase: Phase, - ) { - let _ = request.take(); + ) -> Result<(), Error> { + let active = request.take().ok_or(Error::InvalidArgument)?; self.disable_idmac(); - self.recover_after_idmac_error(phase); + let recovery = self.recover_after_idmac_error(phase); self.clear_all_int_status(); - let _ = slot.complete(id); + self.irq + .state + .clear(crate::DWMMC_INT_COMMAND_DONE | crate::DWMMC_INT_ERROR_MASK); + let completed_dma = self.finish_block_request_with_quiesce(active, recovery.is_ok())?; + drop(completed_dma); + self.pending_data = None; + self.data_blocks_remaining = 0; + self.data_cmd_index = 0; + self.command_state = crate::command::CommandState::Idle; + slot.complete(id)?; + recovery } fn disable_idmac(&self) { @@ -961,7 +1253,7 @@ impl DwMmc { self.regs.bmod().write(0); } - fn recover_after_idmac_error(&mut self, phase: Phase) { + fn recover_after_idmac_error(&mut self, phase: Phase) -> Result<(), Error> { let status = self.regs.status().read().into_bits(); let rintsts = self.regs.rintsts().read(); warn!( @@ -975,16 +1267,24 @@ impl DwMmc { self.regs.ctrl().update(|r| r.with_abort_read_data(true)); let _ = self.regs.ctrl().read(); - let _ = self.reset_fifo(); - let _ = self.reset_dma(); + let fifo = self.reset_fifo(); + let dma = self.reset_dma_for_phase(phase); self.regs.ctrl().update(|r| r.with_abort_read_data(false)); self.pending_data = None; self.data_blocks_remaining = 0; self.data_cmd_index = 0; - } - - fn reset_dma(&self) -> Result<(), Error> { - self.reset_dma_for_phase(Phase::DataRead) + self.command_state = crate::command::CommandState::Idle; + match (fifo, dma) { + (Ok(()), Ok(())) => Ok(()), + (Err(err), _) | (_, Err(err)) => { + self.reset_and_init_preserving_irq()?; + warn!( + "dwmmc: recovered IDMAC {:?} error by controller reset: {err:?}", + phase + ); + Ok(()) + } + } } fn reset_dma_for_phase(&self, phase: Phase) -> Result<(), Error> { @@ -1018,22 +1318,25 @@ impl DwMmc { } fn take_data_irq_status(&mut self) -> u32 { + let consume = crate::DWMMC_INT_DATA_TRANSFER_OVER + | crate::DWMMC_INT_COMMAND_DONE + | crate::DWMMC_INT_RXDR + | crate::DWMMC_INT_TXDR + | crate::DWMMC_INT_ERROR_MASK; + if self.completion_irq_enabled() { + return self.irq.state.take(consume); + } let raw_status = self.regs.rintsts().read().into_bits(); - let consume = raw_status + let clear = raw_status & (crate::DWMMC_INT_DATA_TRANSFER_OVER | crate::DWMMC_INT_COMMAND_DONE | crate::DWMMC_INT_ERROR_MASK); - if consume != 0 { + if clear != 0 { self.regs .rintsts() - .write(crate::regs::RIntSts::from_bits(consume)); + .write(crate::regs::RIntSts::from_bits(clear)); } - let consume = crate::DWMMC_INT_DATA_TRANSFER_OVER - | crate::DWMMC_INT_COMMAND_DONE - | crate::DWMMC_INT_RXDR - | crate::DWMMC_INT_TXDR - | crate::DWMMC_INT_ERROR_MASK; - self.irq_state.take(consume) | raw_status + raw_status } } diff --git a/drivers/blk/dwmmc-host/src/host.rs b/drivers/blk/dwmmc-host/src/host.rs index 87a2ef04c7..3bf84f845d 100644 --- a/drivers/blk/dwmmc-host/src/host.rs +++ b/drivers/blk/dwmmc-host/src/host.rs @@ -9,9 +9,10 @@ //! //! [`SdioHost`]: sdmmc_protocol::sdio::SdioHost +use alloc::sync::Arc; use core::{ ptr::NonNull, - sync::atomic::{AtomicBool, AtomicU32, Ordering}, + sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, }; use dma_api::DeviceDma; @@ -56,47 +57,143 @@ pub(crate) struct PendingData { /// valid, exclusively-owned MMIO base for a DW_mshc-compatible /// register block. Concurrent access to the same controller from /// multiple `DwMmc` instances is undefined. +const IRQ_GENERATION_SHIFT: u64 = 32; +const IRQ_STATUS_MASK: u64 = u32::MAX as u64; + pub(crate) struct IrqState { - pending_status: AtomicU32, + mailbox: AtomicU64, + next_generation: AtomicU32, } impl IrqState { const fn new() -> Self { Self { - pending_status: AtomicU32::new(0), + mailbox: AtomicU64::new(0), + next_generation: AtomicU32::new(0), } } - pub(crate) fn cache(&self, status: u32) { - if status != 0 { - self.pending_status.fetch_or(status, Ordering::AcqRel); + pub(crate) fn begin_request(&self) { + let generation = self.next_generation(); + self.mailbox + .store(pack_mailbox(generation, 0), Ordering::Release); + } + + pub(crate) fn end_request(&self) { + self.mailbox.store(0, Ordering::Release); + } + + pub(crate) fn cache_if_current(&self, generation: u32, status: u32) { + if generation == 0 || status == 0 { + return; + } + let mut cur = self.mailbox.load(Ordering::Acquire); + loop { + if mailbox_generation(cur) != generation { + return; + } + let next = pack_mailbox(generation, mailbox_status(cur) | status); + match self + .mailbox + .compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return, + Err(observed) => cur = observed, + } } } + pub(crate) fn generation(&self) -> u32 { + mailbox_generation(self.mailbox.load(Ordering::Acquire)) + } + pub(crate) fn take(&self, mask: u32) -> u32 { - take_cached_bits(&self.pending_status, mask) + let mut cur = self.mailbox.load(Ordering::Acquire); + loop { + let status = mailbox_status(cur); + let taken = status & mask; + if taken == 0 { + return 0; + } + let next = pack_mailbox(mailbox_generation(cur), status & !mask); + match self + .mailbox + .compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return taken, + Err(observed) => cur = observed, + } + } } pub(crate) fn clear(&self, mask: u32) { - self.pending_status.fetch_and(!mask, Ordering::AcqRel); + let mut cur = self.mailbox.load(Ordering::Acquire); + loop { + let next = pack_mailbox(mailbox_generation(cur), mailbox_status(cur) & !mask); + match self + .mailbox + .compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return, + Err(observed) => cur = observed, + } + } } #[cfg(test)] pub(crate) fn pending(&self) -> u32 { - self.pending_status.load(Ordering::Acquire) + mailbox_status(self.mailbox.load(Ordering::Acquire)) } -} -fn take_cached_bits(cache: &AtomicU32, mask: u32) -> u32 { - let mut cur = cache.load(Ordering::Acquire); - loop { - let taken = cur & mask; - if taken == 0 { - return 0; + fn next_generation(&self) -> u32 { + let mut cur = self.next_generation.load(Ordering::Acquire); + loop { + let mut next = cur.wrapping_add(1); + if next == 0 { + next = 1; + } + match self.next_generation.compare_exchange_weak( + cur, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return next, + Err(observed) => cur = observed, + } } - match cache.compare_exchange_weak(cur, cur & !mask, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => return taken, - Err(next) => cur = next, + } +} + +fn pack_mailbox(generation: u32, status: u32) -> u64 { + ((generation as u64) << IRQ_GENERATION_SHIFT) | status as u64 +} + +fn mailbox_generation(value: u64) -> u32 { + (value >> IRQ_GENERATION_SHIFT) as u32 +} + +fn mailbox_status(value: u64) -> u32 { + (value & IRQ_STATUS_MASK) as u32 +} + +pub(crate) struct IrqCore { + pub(crate) regs: VolatilePtr<'static, RegisterBlock>, + pub(crate) state: IrqState, +} + +// SAFETY: `IrqCore` is shared only between the task-side host and the IRQ +// top-half. Both access the register block with volatile operations and share +// interrupt status through atomics. +unsafe impl Send for IrqCore {} +// SAFETY: See the `Send` impl. +unsafe impl Sync for IrqCore {} + +impl IrqCore { + fn new(regs: VolatilePtr<'static, RegisterBlock>) -> Self { + Self { + regs, + state: IrqState::new(), } } } @@ -112,8 +209,11 @@ pub struct DwMmc { pub(crate) data_cmd_index: u8, pub(crate) dma: Option, pub(crate) dma_mask: u64, - pub(crate) irq_state: IrqState, + pub(crate) dma_poisoned: bool, + pub(crate) irq: Arc, pub(crate) completion_irq_enabled: AtomicBool, + pub(crate) host2_next_id: u64, + pub(crate) host2_active_id: Option, } impl DwMmc { @@ -151,8 +251,11 @@ impl DwMmc { data_cmd_index: 0, dma: None, dma_mask: u32::MAX as u64, - irq_state: IrqState::new(), + dma_poisoned: false, + irq: Arc::new(IrqCore::new(regs)), completion_irq_enabled: AtomicBool::new(false), + host2_next_id: 0, + host2_active_id: None, } } @@ -227,6 +330,18 @@ impl DwMmc { self.dma = Some(dma); } + pub(crate) fn check_not_poisoned(&self) -> Result<(), Error> { + if self.dma_poisoned { + Err(Error::BusError(ErrorContext::new(Phase::DataRead))) + } else { + Ok(()) + } + } + + pub(crate) fn poison_dma(&mut self) { + self.dma_poisoned = true; + } + /// Bring the controller to a known state and arm it for card /// identification at 400 kHz. /// @@ -263,7 +378,7 @@ impl DwMmc { // Mask every interrupt; clear any leftover raw status. self.regs.intmask().write(0); self.clear_all_int_status(); - self.irq_state.clear(u32::MAX); + self.irq.state.clear(u32::MAX); self.completion_irq_enabled.store(false, Ordering::Release); // Default to 1-bit bus until the protocol layer asks for wider. @@ -273,6 +388,16 @@ impl DwMmc { // Program the divider for 400 kHz (the SD spec ID-mode rate). self.program_clock(400_000)?; + self.dma_poisoned = false; + Ok(()) + } + + pub(crate) fn reset_and_init_preserving_irq(&mut self) -> Result<(), Error> { + let was_irq_enabled = self.completion_irq_enabled(); + self.reset_and_init()?; + if was_irq_enabled { + self.enable_completion_irq(); + } Ok(()) } diff --git a/drivers/blk/dwmmc-host/src/lib.rs b/drivers/blk/dwmmc-host/src/lib.rs index b68f16d2ee..c5ef8dbd76 100644 --- a/drivers/blk/dwmmc-host/src/lib.rs +++ b/drivers/blk/dwmmc-host/src/lib.rs @@ -1,7 +1,7 @@ //! Synopsys DesignWare Mobile Storage Host Controller (DW_mshc) backend //! for the [`sdmmc-protocol`](sdmmc_protocol) driver crate. //! -//! Implements [`sdmmc_protocol::sdio::SdioHost`] for the IP block known +//! Implements [`sdio_host2::SdioHost`] for the IP block known //! variously as DWC_mobile_storage, dw_mshc, dw_mmc (Linux), or simply //! the "Synopsys SD/MMC controller" — the same core used in Rockchip //! RK33xx/RK35xx, Allwinner A-series, StarFive JH7110, and a long @@ -32,9 +32,10 @@ //! let mmio = NonNull::new(0xFE2B_0000 as *mut u8).unwrap(); //! let mut host = unsafe { DwMmc::new(mmio) }; //! host.set_reference_clock(50_000_000); -//! host.reset_and_init().expect("controller reset"); +//! // Optional DMA capability can be installed here before the protocol layer +//! // owns the host. //! -//! let mut card = SdioSdmmc::new(host); +//! let mut card = SdioSdmmc::new_host2(host); //! let mut scratch = SdioInitScratch::new(); //! let mut request = card.submit_init(&mut scratch)?; //! // Poll request here. Runtime code chooses spin, yield, IRQ wait, or timer. @@ -55,11 +56,15 @@ #![no_std] #![allow(clippy::missing_safety_doc)] +extern crate alloc; + +use alloc::sync::Arc; use core::{marker::PhantomData, num::NonZeroUsize, ptr::NonNull}; mod command; mod dma; mod host; +pub mod rdif; mod regs; pub use sdmmc_protocol::block::{ @@ -67,12 +72,13 @@ pub use sdmmc_protocol::block::{ BlockTransferState, }; use sdmmc_protocol::{ - DataCommandPoll, + DataCommandPoll, OperationPoll, cmd::{Command, DataDirection}, - error::Error, + error::{Error, ErrorContext, Phase}, sdio::{ - BusWidth, ClockSpeed, HostEvent, HostEventKind, HostEventSource, SdioHost, SdioIrqHandle, - SdioIrqHost, SignalVoltage, + BusWidth, ClockSpeed, HostEvent, HostEventKind, HostEventSource, ReadyBusRequest, + SdioBusOp, SdioHost as ProtocolSdioHost, SdioIrqHandle, SdioIrqHost, SignalVoltage, + poll_ready_bus_op, submit_ready_bus_op, }, }; @@ -109,19 +115,110 @@ pub struct DataRequest<'a> { _buffer: PhantomData<&'a [u8]>, } +pub struct TransactionRequest<'a> { + owner: usize, + id: u64, + done: bool, + kind: TransactionRequestKind, + data: Option>, +} + +enum TransactionRequestKind { + Command { response: sdio_host2::ResponseType }, + Data { response: sdio_host2::ResponseType }, +} + +impl<'a> TransactionRequest<'a> { + fn command(owner: usize, id: u64, response: sdio_host2::ResponseType) -> Self { + Self { + owner, + id, + done: false, + kind: TransactionRequestKind::Command { response }, + data: None, + } + } + + fn data( + owner: usize, + id: u64, + request: DataRequest<'a>, + response: sdio_host2::ResponseType, + ) -> Self { + Self { + owner, + id, + done: false, + kind: TransactionRequestKind::Data { response }, + data: Some(request), + } + } +} + +pub struct BusRequest { + owner: usize, + id: u64, + done: bool, + state: BusRequestState, +} + +impl BusRequest { + fn pending(owner: usize, id: u64, state: BusRequestState) -> Self { + Self { + owner, + id, + done: false, + state, + } + } +} + +enum BusRequestState { + ResetAll(DwMmcResetState), + ResetDataLine { started: bool, polls: u32 }, + PowerOn, + PowerOff, + SetClock(DwMmcClockState), + SetBusWidth(BusWidth), + SetSignalVoltage(SignalVoltage), +} + +enum DwMmcResetState { + Start, + WaitReset { polls: u32 }, + InitClock(DwMmcClockState), +} + +enum DwMmcClockState { + Start { + speed: Option, + target_hz: u32, + }, + WaitGate { + polls: u32, + target_hz: u32, + }, + ProgramDivider { + target_hz: u32, + }, + WaitDivider { + polls: u32, + }, + Enable, + WaitEnable { + polls: u32, + }, +} + +const DWMMC_RESET_POLLS: u32 = 1_000_000; +const DWMMC_CLOCK_POLLS: u32 = 1_000_000; + /// Cloneable, sync-safe DWMMC IRQ top-half handle. #[derive(Clone)] pub struct DwMmcIrqHandle { - regs: volatile::VolatilePtr<'static, crate::regs::RegisterBlock>, - irq_state: *const host::IrqState, + irq: Arc, } -// SAFETY: The handle only performs volatile MMIO accesses and atomic cache -// updates. The owning `DwMmc` outlives handles created by OS glue. -unsafe impl Send for DwMmcIrqHandle {} -// SAFETY: See the `Send` impl. -unsafe impl Sync for DwMmcIrqHandle {} - pub(crate) const DWMMC_INT_RESPONSE_ERROR: u32 = 1 << 1; pub(crate) const DWMMC_INT_COMMAND_DONE: u32 = 1 << 2; pub(crate) const DWMMC_INT_DATA_TRANSFER_OVER: u32 = 1 << 3; @@ -147,11 +244,13 @@ pub(crate) const DWMMC_INT_ERROR_MASK: u32 = DWMMC_INT_RESPONSE_ERROR | DWMMC_INT_START_BIT_ERROR | DWMMC_INT_END_BIT_ERROR; -impl SdioHost for DwMmc { +impl ProtocolSdioHost for DwMmc { type Event = Event; type DataRequest<'a> = DataRequest<'a>; + type BusRequest = ReadyBusRequest; fn submit_command(&mut self, cmd: &Command) -> Result<(), Error> { + self.check_not_poisoned()?; DwMmc::submit_command(self, cmd) } @@ -248,6 +347,14 @@ impl SdioHost for DwMmc { fn handle_irq(&mut self) -> Self::Event { self.irq_handle().handle_irq() } + + fn submit_bus_op(&mut self, op: SdioBusOp) -> Result { + submit_ready_bus_op(self, op) + } + + fn poll_bus_op(&mut self, request: &mut Self::BusRequest) -> Result, Error> { + poll_ready_bus_op(request) + } } impl SdioIrqHost for DwMmc { @@ -262,6 +369,699 @@ impl SdioIrqHost for DwMmc { } } +impl sdio_host2::SdioHost for DwMmc { + type TransactionRequest<'a> + = TransactionRequest<'a> + where + Self: 'a; + type BusRequest = BusRequest; + + unsafe fn submit_transaction<'a>( + &mut self, + transaction: sdio_host2::Transaction<'a>, + ) -> Result, sdio_host2::Error> + where + Self: 'a, + { + self.check_not_poisoned().map_err(map_protocol_error)?; + if !self.physical_bus_idle() { + return Err(sdio_host2::Error::Busy); + } + let owner = self.host2_owner(); + let id = self.start_host2_request(); + let response = transaction.command.response; + match transaction.data { + None => { + if let Err(err) = self.submit_command(&transaction.command) { + self.finish_host2_request(id); + return Err(map_protocol_error(err)); + } + Ok(TransactionRequest::command(owner, id, response)) + } + Some(phase) => { + phase + .validate() + .inspect_err(|_| self.finish_host2_request(id))?; + let block_size = u32::from(phase.block_size.get()); + let block_count = phase.block_count.get(); + let request = match phase.buffer { + sdio_host2::DataBuffer::Read(buf) => { + if !matches!(phase.direction, sdio_host2::DataDirection::Read) { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + ::submit_read_data( + self, + &transaction.command, + buf, + block_size, + block_count, + ) + } + sdio_host2::DataBuffer::Write(buf) => { + if !matches!(phase.direction, sdio_host2::DataDirection::Write) { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + ::submit_write_data( + self, + &transaction.command, + buf, + block_size, + block_count, + ) + } + sdio_host2::DataBuffer::Dma(_) => { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + } + .inspect_err(|_| self.finish_host2_request(id)) + .map_err(map_protocol_error)?; + Ok(TransactionRequest::data(owner, id, request, response)) + } + } + } + + unsafe fn submit_transaction_owned<'a>( + &mut self, + transaction: sdio_host2::Transaction<'a>, + ) -> Result, sdio_host2::SubmitTransactionError<'a>> + where + Self: 'a, + { + if let Err(err) = self.check_not_poisoned() { + return Err(sdio_host2::SubmitTransactionError::new( + map_protocol_error(err), + transaction, + )); + } + if !matches!( + transaction.data.as_ref().map(|data| &data.buffer), + Some(sdio_host2::DataBuffer::Dma(_)) + ) { + return unsafe { self.submit_transaction(transaction) } + .map_err(sdio_host2::SubmitTransactionError::consumed); + } + if !self.physical_bus_idle() { + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Busy, + transaction, + )); + } + + let owner = self.host2_owner(); + let host2_id = self.start_host2_request(); + let response = transaction.command.response; + let Some(phase) = transaction.data else { + unreachable!("DMA transaction must contain a data phase") + }; + let block_size = u32::from(phase.block_size.get()); + let block_count = phase.block_count.get(); + let sdio_host2::DataBuffer::Dma(buffer) = phase.buffer else { + unreachable!("checked for DMA data buffer above") + }; + if !should_try_dma( + &transaction.command, + block_size, + block_count, + buffer.len().get(), + match phase.direction { + sdio_host2::DataDirection::Read => DataDirection::Read, + sdio_host2::DataDirection::Write => DataDirection::Write, + _ => { + self.finish_host2_request(host2_id); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + sdio_host2::Transaction::with_data(transaction.command, data), + )); + } + }, + ) { + self.finish_host2_request(host2_id); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + sdio_host2::Transaction::with_data(transaction.command, data), + )); + } + let Some(dma) = self.dma.clone() else { + self.finish_host2_request(host2_id); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + sdio_host2::Transaction::with_data(transaction.command, data), + )); + }; + let mut slot = BlockRequestSlot::default(); + let submit = match phase.direction { + sdio_host2::DataDirection::Read => self.submit_prepared_read_blocks( + transaction.command.argument, + buffer, + &dma, + &mut slot, + ), + sdio_host2::DataDirection::Write => self.submit_prepared_write_blocks( + transaction.command.argument, + buffer, + &dma, + &mut slot, + ), + _ => unreachable!("unsupported direction returned before submit"), + }; + match submit { + Ok(request) => { + let id = request.id(); + let data = DataRequest { + id, + request: Some(request), + slot, + _buffer: PhantomData, + }; + Ok(TransactionRequest::data(owner, host2_id, data, response)) + } + Err(err) => { + self.finish_host2_request(host2_id); + let error = err.error; + let buffer = err.into_buffer(); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + Err(sdio_host2::SubmitTransactionError::new( + map_protocol_error(error), + sdio_host2::Transaction::with_data(transaction.command, data), + )) + } + } + } + + fn poll_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result, sdio_host2::PollRequestError> + where + Self: 'a, + { + self.check_host2_transaction_request(request)?; + match request.kind { + TransactionRequestKind::Command { response } => { + match ::poll_command_response(self) { + Ok(sdmmc_protocol::CommandResponsePoll::Pending) => { + Ok(sdio_host2::RequestPoll::Pending) + } + Ok(sdmmc_protocol::CommandResponsePoll::Complete(resp)) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok( + resp.to_raw_response(response) + ))) + } + Ok(_) => Ok(sdio_host2::RequestPoll::Pending), + Err(err) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(map_protocol_error(err)))) + } + } + } + TransactionRequestKind::Data { response } => { + let Some(data) = request.data.as_mut() else { + let recovery = self.abort_host2_transaction_request(request).err(); + return Ok(sdio_host2::RequestPoll::Ready(Err( + recovery.unwrap_or(sdio_host2::Error::InvalidArgument) + ))); + }; + match ::poll_data_request(self, data) { + Ok(DataCommandPoll::Pending) => Ok(sdio_host2::RequestPoll::Pending), + Ok(DataCommandPoll::Complete(resp)) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok( + resp.to_raw_response(response) + ))) + } + Ok(_) => Ok(sdio_host2::RequestPoll::Pending), + Err(err) => { + let _ = self.abort_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(map_protocol_error(err)))) + } + } + } + } + } + + fn abort_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result<(), sdio_host2::Error> + where + Self: 'a, + { + if request.done { + return Ok(()); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::Error::InvalidArgument); + } + self.abort_host2_transaction_request(request) + } + + fn take_completed_dma<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Option + where + Self: 'a, + { + request + .data + .as_mut() + .and_then(|data| data.slot.take_completed_dma()) + } + + unsafe fn submit_bus_op( + &mut self, + op: sdio_host2::BusOp, + ) -> Result { + self.check_not_poisoned().map_err(map_protocol_error)?; + if !self.physical_bus_idle() { + return Err(sdio_host2::Error::Busy); + } + let state = self.prepare_host2_bus_op(op)?; + let owner = self.host2_owner(); + let id = self.start_host2_request(); + Ok(BusRequest::pending(owner, id, state)) + } + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, sdio_host2::PollRequestError> { + self.check_host2_bus_request(request)?; + match self.poll_host2_bus_state(&mut request.state) { + Ok(sdio_host2::RequestPoll::Pending) => Ok(sdio_host2::RequestPoll::Pending), + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) => { + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + Ok(sdio_host2::RequestPoll::Ready(Err(err))) => { + let _ = self.abort_host2_bus_state(&mut request.state); + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(err))) + } + Err(err) => { + let _ = self.abort_host2_bus_state(&mut request.state); + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(err))) + } + } + } + + fn abort_bus_op(&mut self, request: &mut Self::BusRequest) -> Result<(), sdio_host2::Error> { + if request.done { + return Ok(()); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::Error::InvalidArgument); + } + let result = self.abort_host2_bus_state(&mut request.state); + request.done = true; + self.finish_host2_request(request.id); + result + } +} + +impl DwMmc { + fn physical_bus_idle(&self) -> bool { + matches!(self.command_state, command::CommandState::Idle) + && self.pending_data.is_none() + && self.data_blocks_remaining == 0 + && self.host2_active_id.is_none() + } + + fn start_host2_request(&mut self) -> u64 { + let id = self.host2_next_id; + self.host2_next_id = self.host2_next_id.wrapping_add(1); + self.host2_active_id = Some(id); + id + } + + fn host2_owner(&self) -> usize { + self.base_addr + } + + fn finish_host2_request(&mut self, id: u64) { + if self.host2_active_id == Some(id) { + self.host2_active_id = None; + } + } + + fn prepare_host2_bus_op( + &self, + op: sdio_host2::BusOp, + ) -> Result { + match op { + sdio_host2::BusOp::ResetAll => Ok(BusRequestState::ResetAll(DwMmcResetState::Start)), + sdio_host2::BusOp::ResetCommandLine => Err(sdio_host2::Error::Unsupported), + sdio_host2::BusOp::ResetDataLine => Ok(BusRequestState::ResetDataLine { + started: false, + polls: 0, + }), + sdio_host2::BusOp::PowerOn => Ok(BusRequestState::PowerOn), + sdio_host2::BusOp::PowerOff => Ok(BusRequestState::PowerOff), + sdio_host2::BusOp::SetClock(speed) => { + let target_hz = clock_hz_for_speed(speed); + if target_hz == 0 { + return Err(sdio_host2::Error::Unsupported); + } + Ok(BusRequestState::SetClock(DwMmcClockState::Start { + speed: Some(speed), + target_hz, + })) + } + sdio_host2::BusOp::SetClockHz(sdio_host2::ClockHz(hz)) => { + Ok(BusRequestState::SetClock(DwMmcClockState::Start { + speed: None, + target_hz: hz, + })) + } + sdio_host2::BusOp::SetBusWidth(width) => match width { + BusWidth::Bit1 | BusWidth::Bit4 | BusWidth::Bit8 => { + Ok(BusRequestState::SetBusWidth(width)) + } + _ => Err(sdio_host2::Error::Unsupported), + }, + sdio_host2::BusOp::SetSignalVoltage(voltage) => match volt_mask_for_signal(voltage) { + Ok(_) => Ok(BusRequestState::SetSignalVoltage(voltage)), + Err(err) => Err(map_protocol_error(err)), + }, + sdio_host2::BusOp::ExecuteTuning { .. } => Err(sdio_host2::Error::Unsupported), + _ => Err(sdio_host2::Error::Unsupported), + } + } + + fn poll_host2_bus_state( + &mut self, + state: &mut BusRequestState, + ) -> Result, sdio_host2::Error> { + match state { + BusRequestState::ResetAll(reset) => self.poll_host2_reset_all(reset), + BusRequestState::ResetDataLine { started, polls } => { + self.poll_host2_fifo_reset(started, polls) + } + BusRequestState::PowerOn => { + self.regs.pwren().write(1); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::PowerOff => { + self.regs.pwren().write(0); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::SetClock(clock) => self.poll_host2_clock(clock), + BusRequestState::SetBusWidth(width) => { + self.set_card_type(*width); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::SetSignalVoltage(voltage) => { + self.set_signal_voltage(*voltage) + .map_err(map_protocol_error)?; + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + } + } + + fn poll_host2_reset_all( + &mut self, + state: &mut DwMmcResetState, + ) -> Result, sdio_host2::Error> { + match state { + DwMmcResetState::Start => { + self.regs.clkena().write(crate::regs::ClkEna::new()); + self.regs.ctrl().update(|r| { + r.with_use_internal_dmac(false) + .with_dma_enable(false) + .with_int_enable(false) + }); + self.regs.ctrl().update(|r| { + r.with_controller_reset(true) + .with_fifo_reset(true) + .with_dma_reset(true) + }); + *state = DwMmcResetState::WaitReset { polls: 0 }; + Ok(sdio_host2::RequestPoll::Pending) + } + DwMmcResetState::WaitReset { polls } => { + let ctrl = self.regs.ctrl().read(); + if !ctrl.controller_reset() && !ctrl.fifo_reset() && !ctrl.dma_reset() { + self.regs.intmask().write(0); + self.clear_all_int_status(); + self.irq.state.clear(u32::MAX); + self.completion_irq_enabled + .store(false, core::sync::atomic::Ordering::Release); + self.regs.ctype().write(crate::regs::CType::new()); + self.regs.uhs().write(crate::regs::UHS::new()); + *state = DwMmcResetState::InitClock(DwMmcClockState::Start { + speed: None, + target_hz: 400_000, + }); + return Ok(sdio_host2::RequestPoll::Pending); + } + if *polls >= DWMMC_RESET_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new( + Phase::Init, + )))); + } + *polls += 1; + Ok(sdio_host2::RequestPoll::Pending) + } + DwMmcResetState::InitClock(clock) => self.poll_host2_clock(clock), + } + } + + fn poll_host2_fifo_reset( + &mut self, + started: &mut bool, + polls: &mut u32, + ) -> Result, sdio_host2::Error> { + if !*started { + self.regs.ctrl().update(|r| r.with_fifo_reset(true)); + *started = true; + } + if !self.regs.ctrl().read().fifo_reset() { + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + if *polls >= DWMMC_RESET_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new( + Phase::DataRead, + )))); + } + *polls += 1; + Ok(sdio_host2::RequestPoll::Pending) + } + + fn poll_host2_clock( + &mut self, + state: &mut DwMmcClockState, + ) -> Result, sdio_host2::Error> { + match state { + DwMmcClockState::Start { speed, target_hz } => { + if let Some(speed) = *speed { + self.set_uhs_timing(speed); + } + self.regs.clkena().write(crate::regs::ClkEna::new()); + self.start_update_clock(false); + *state = DwMmcClockState::WaitGate { + polls: 0, + target_hz: *target_hz, + }; + Ok(sdio_host2::RequestPoll::Pending) + } + DwMmcClockState::WaitGate { polls, target_hz } => { + if self.poll_update_clock_complete(polls)? { + *state = DwMmcClockState::ProgramDivider { + target_hz: *target_hz, + }; + } + Ok(sdio_host2::RequestPoll::Pending) + } + DwMmcClockState::ProgramDivider { target_hz } => { + let div = dwmmc_clock_divisor(self.ref_clock_hz, *target_hz); + self.regs + .clkdiv() + .write(crate::regs::ClkDiv::new().with_clk_divider0(div)); + self.start_update_clock(false); + *state = DwMmcClockState::WaitDivider { polls: 0 }; + Ok(sdio_host2::RequestPoll::Pending) + } + DwMmcClockState::WaitDivider { polls } => { + if self.poll_update_clock_complete(polls)? { + *state = DwMmcClockState::Enable; + } + Ok(sdio_host2::RequestPoll::Pending) + } + DwMmcClockState::Enable => { + self.regs + .clkena() + .write(crate::regs::ClkEna::new().with_cclk_enable(1)); + self.start_update_clock(false); + *state = DwMmcClockState::WaitEnable { polls: 0 }; + Ok(sdio_host2::RequestPoll::Pending) + } + DwMmcClockState::WaitEnable { polls } => { + if self.poll_update_clock_complete(polls)? { + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + Ok(sdio_host2::RequestPoll::Pending) + } + } + } + + fn start_update_clock(&self, voltage_switch: bool) { + self.regs.cmd().write( + crate::regs::Cmd::new() + .with_start_cmd(true) + .with_wait_prvdata_complete(true) + .with_update_clock_registers_only(true) + .with_volt_switch(voltage_switch), + ); + } + + fn poll_update_clock_complete(&self, polls: &mut u32) -> Result { + if !self.regs.cmd().read().start_cmd() { + return Ok(true); + } + if *polls >= DWMMC_CLOCK_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new( + Phase::Init, + )))); + } + *polls += 1; + Ok(false) + } + + fn check_host2_transaction_request( + &self, + request: &TransactionRequest<'_>, + ) -> Result<(), sdio_host2::PollRequestError> { + if request.done { + return Err(sdio_host2::PollRequestError::AlreadyCompleted); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::PollRequestError::WrongOwner); + } + if self.host2_active_id != Some(request.id) { + return Err(sdio_host2::PollRequestError::StaleGeneration); + } + Ok(()) + } + + fn check_host2_bus_request( + &self, + request: &BusRequest, + ) -> Result<(), sdio_host2::PollRequestError> { + if request.done { + return Err(sdio_host2::PollRequestError::AlreadyCompleted); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::PollRequestError::WrongOwner); + } + if self.host2_active_id != Some(request.id) { + return Err(sdio_host2::PollRequestError::StaleGeneration); + } + Ok(()) + } + + fn complete_host2_transaction_request(&mut self, request: &mut TransactionRequest<'_>) { + request.done = true; + self.finish_host2_request(request.id); + } + + fn complete_host2_bus_request(&mut self, request: &mut BusRequest) { + request.done = true; + self.finish_host2_request(request.id); + } + + fn abort_host2_bus_state( + &mut self, + state: &mut BusRequestState, + ) -> Result<(), sdio_host2::Error> { + match state { + BusRequestState::ResetAll(_) + | BusRequestState::SetClock(_) + | BusRequestState::SetSignalVoltage(_) => { + self.reset_and_init_preserving_irq() + .map_err(map_protocol_error)?; + } + BusRequestState::ResetDataLine { started, .. } if *started => { + self.reset_fifo().map_err(map_protocol_error)?; + } + BusRequestState::PowerOn + | BusRequestState::PowerOff + | BusRequestState::SetBusWidth(_) => {} + BusRequestState::ResetDataLine { .. } => {} + } + self.pending_data = None; + self.data_blocks_remaining = 0; + self.command_state = command::CommandState::Idle; + Ok(()) + } + + fn abort_host2_transaction_request( + &mut self, + request: &mut TransactionRequest<'_>, + ) -> Result<(), sdio_host2::Error> { + let result = if let Some(data) = request.data.as_mut() { + if let Some(active) = data.request.take() { + let id = active.id(); + let mut pending = Some(active); + self.abort_block_request_response(&mut pending, id, &mut data.slot) + .map_err(map_protocol_error) + } else { + Ok(()) + } + } else { + self.abort_command().map_err(map_protocol_error) + }; + request.done = true; + self.finish_host2_request(request.id); + result + } +} + +fn map_protocol_error(err: Error) -> sdio_host2::Error { + match err { + Error::Timeout(_) => sdio_host2::Error::Timeout, + Error::Crc(_) => sdio_host2::Error::Crc, + Error::NoCard => sdio_host2::Error::NoCard, + Error::Busy => sdio_host2::Error::Busy, + Error::UnsupportedCommand => sdio_host2::Error::Unsupported, + Error::Misaligned => sdio_host2::Error::Misaligned, + Error::InvalidArgument => sdio_host2::Error::InvalidArgument, + Error::BusError(_) => sdio_host2::Error::Bus, + Error::ReadError(_) | Error::WriteError(_) | Error::BadResponse(_) => { + sdio_host2::Error::Bus + } + Error::CardError(_) | Error::CardLocked => sdio_host2::Error::Controller, + _ => sdio_host2::Error::Controller, + } +} + fn submit_read_with_dma_fifo_fallback( host: &mut DwMmc, cmd: &Command, @@ -275,7 +1075,7 @@ fn submit_read_with_dma_fifo_fallback( && let Some(dma) = host.dma.clone() { match host.submit_read_blocks( - cmd.arg, + cmd.argument, buffer, NonZeroUsize::new(len).ok_or(Error::InvalidArgument)?, Some(&dma), @@ -312,7 +1112,7 @@ fn submit_write_with_dma_fifo_fallback( && let Some(dma) = host.dma.clone() { match host.submit_write_blocks( - cmd.arg, + cmd.argument, buffer, NonZeroUsize::new(len).ok_or(Error::InvalidArgument)?, Some(&dma), @@ -346,7 +1146,7 @@ fn should_try_dma( block_size == 512 && len == block_count as usize * 512 && matches!( - (direction, cmd.cmd), + (direction, cmd.index), (DataDirection::Read, 17 | 18) | (DataDirection::Write, 24 | 25) ) } @@ -428,8 +1228,7 @@ impl DwMmc { pub fn irq_handle(&self) -> DwMmcIrqHandle { DwMmcIrqHandle { - regs: self.regs, - irq_state: &self.irq_state, + irq: self.irq.clone(), } } @@ -444,13 +1243,15 @@ impl SdioIrqHandle for DwMmcIrqHandle { type Event = Event; fn handle_irq(&self) -> Self::Event { - let raw_status = self.regs.mintsts().read(); + let generation = self.irq.state.generation(); + let raw_status = self.irq.regs.mintsts().read(); if raw_status != 0 { - self.regs + self.irq + .regs .rintsts() .write(crate::regs::RIntSts::from_bits(raw_status)); } - unsafe { &*self.irq_state }.cache(raw_status); + self.irq.state.cache_if_current(generation, raw_status); event_from_raw_status(raw_status) } } @@ -468,6 +1269,14 @@ fn clock_hz_for_speed(speed: ClockSpeed) -> u32 { } } +fn dwmmc_clock_divisor(ref_clock_hz: u32, target_hz: u32) -> u8 { + if ref_clock_hz == 0 || target_hz == 0 || target_hz >= ref_clock_hz { + 0 + } else { + ref_clock_hz.div_ceil(2 * target_hz).min(0xFF) as u8 + } +} + pub(crate) fn ddr_mask_for_speed(speed: ClockSpeed) -> u16 { match speed { ClockSpeed::Ddr50 => 1, @@ -510,6 +1319,10 @@ pub(crate) fn uhs_bits_after_voltage( #[cfg(test)] mod tests { + use core::num::{NonZeroU16, NonZeroU32}; + + use sdio_host2::ResponseType; + use super::*; #[test] @@ -563,11 +1376,40 @@ mod tests { assert_eq!(dma.dma_mask, Some(u32::MAX as u64)); } + #[test] + fn host2_data_submit_reports_busy_without_dirtying_pending_data() { + let mut host = unsafe { DwMmc::new_from_addr(0x1000_0000) }; + host.command_state = command::CommandState::Issued { + cmd: Command::new(0, 0, ResponseType::None), + polls: 0, + }; + let mut buf = [0u8; 512]; + let data = sdio_host2::DataPhase::read( + NonZeroU16::new(512).unwrap(), + NonZeroU32::new(1).unwrap(), + &mut buf, + ) + .unwrap(); + let tx = sdio_host2::Transaction::with_data(Command::new(17, 0, ResponseType::R1), data); + + let err = + match unsafe { ::submit_transaction(&mut host, tx) } { + Ok(_) => panic!("busy host accepted a second transaction"), + Err(err) => err, + }; + + assert_eq!(err, sdio_host2::Error::Busy); + assert!(host.pending_data.is_none()); + assert_eq!(host.data_blocks_remaining, 0); + } + #[test] fn irq_handle_acks_and_caches_status_without_mutable_host() { let mut mmio = [0u32; 256]; let base = NonNull::new(mmio.as_mut_ptr().cast()).unwrap(); let host = unsafe { DwMmc::new(base) }; + host.irq.state.begin_request(); + let old_generation = host.irq.state.generation(); let raw = crate::regs::RIntSts::new() .with_data_transfer_over(true) .into_bits(); @@ -579,11 +1421,19 @@ mod tests { let handle = host.irq_handle().clone(); assert_eq!(handle.handle_irq(), Event::TransferComplete); - assert_eq!(host.irq_state.pending(), raw); + assert_eq!(host.irq.state.pending(), raw); unsafe { mmio.as_mut_ptr().add(MINTSTS_WORD).write_volatile(0); } assert_eq!(host.handle_irq(), Event::None); + + host.irq.state.end_request(); + host.irq.state.begin_request(); + assert_ne!(host.irq.state.generation(), old_generation); + host.irq + .state + .cache_if_current(old_generation, crate::DWMMC_INT_DATA_TRANSFER_OVER); + assert_eq!(host.irq.state.pending(), 0); } #[test] diff --git a/drivers/blk/dwmmc-host/src/rdif.rs b/drivers/blk/dwmmc-host/src/rdif.rs new file mode 100644 index 0000000000..8df436ca4c --- /dev/null +++ b/drivers/blk/dwmmc-host/src/rdif.rs @@ -0,0 +1,56 @@ +//! RDIF block-device adapter for [`DwMmc`]. + +use dma_api::DeviceDma; +pub use protocol_rdif::{BlockConfig, BlockDevice, BlockQueue}; +pub use rdif_block::{ + BInterface, BIrqHandler, BOwnedQueue, BQueue, BlkError, IQueue, IQueueOwned, Interface, + IrqHandlerHandle, IrqHandlerSlot, OwnedRequest, PollError, QueueHandle, Request, + RequestId as RdifRequestId, RequestPoll as OwnedRequestPoll, RequestStatus, SubmitError, +}; +use sdmmc_protocol::{ + rdif as protocol_rdif, + sdio::{SdioHost2Adapter, SdioSdmmc}, +}; + +use crate::DwMmc; + +pub fn device( + card: SdioSdmmc>, + config: BlockConfig, +) -> BlockDevice> { + BlockDevice::new(card, config) +} + +pub fn dma_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, + dma: DeviceDma, +) -> BlockConfig { + BlockConfig::dma(name, capacity_blocks, irq_driven, dma) + .with_max_blocks_per_request(1024) + .with_max_segment_size(1024 * protocol_rdif::BLOCK_SIZE) +} + +pub const fn fifo_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, +) -> BlockConfig { + BlockConfig::fifo(name, capacity_blocks, irq_driven) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fifo_config_keeps_one_block_limits() { + let config = fifo_config("dwmmc", 16, true); + let limits = protocol_rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, 1); + assert_eq!(limits.max_segment_size, protocol_rdif::BLOCK_SIZE); + assert!(!config.uses_dma()); + } +} diff --git a/drivers/blk/nvme-driver/src/block.rs b/drivers/blk/nvme-driver/src/block.rs index b9a9ab4bb5..524e709a1b 100644 --- a/drivers/blk/nvme-driver/src/block.rs +++ b/drivers/blk/nvme-driver/src/block.rs @@ -867,6 +867,7 @@ fn limits( let max_bytes = (max_blocks as usize).saturating_mul(lba_size); QueueLimits { dma_mask, + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_alignment, max_inflight: max_inflight.max(1), max_blocks_per_request: max_blocks, diff --git a/drivers/blk/nvme-driver/src/nvme.rs b/drivers/blk/nvme-driver/src/nvme.rs index f8804b96d4..2aed63da0d 100644 --- a/drivers/blk/nvme-driver/src/nvme.rs +++ b/drivers/blk/nvme-driver/src/nvme.rs @@ -66,7 +66,7 @@ impl Nvme { ) -> Result { mmio_api::init(mmio_op); let mmio = mmio_api::ioremap(bar_addr.into(), bar_size)?; - let dma = DeviceDma::new(dma_mask, dma_op); + let dma = DeviceDma::new_legacy(dma_mask, dma_op); Self::new_mmio(mmio, dma, config) } diff --git a/drivers/blk/phytium-mci-host/Cargo.toml b/drivers/blk/phytium-mci-host/Cargo.toml index 8ff9b26319..83ceaf6755 100644 --- a/drivers/blk/phytium-mci-host/Cargo.toml +++ b/drivers/blk/phytium-mci-host/Cargo.toml @@ -10,12 +10,14 @@ keywords = ["sd", "mmc", "phytium", "embedded", "no_std"] categories = ["embedded", "no-std", "hardware-support"] [dependencies] -sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio"] } +sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio", "rdif"] } +sdio-host2.workspace = true bitfield-struct = "0.11" volatile = { version = "0.6", features = ["derive"] } log.workspace = true dma-api.workspace = true mmio-api.workspace = true +rdif-block.workspace = true [features] default = [] diff --git a/drivers/blk/phytium-mci-host/README.md b/drivers/blk/phytium-mci-host/README.md index d5d8562e67..9b4ba4ba6e 100644 --- a/drivers/blk/phytium-mci-host/README.md +++ b/drivers/blk/phytium-mci-host/README.md @@ -7,3 +7,10 @@ The crate owns register programming, command/response handling, FIFO and IDMAC block transfers, clock timing selection, and IRQ event extraction. Platform code still owns FDT/ACPI probe, MMIO mapping lifetime, IRQ registration, pad-controller setup, and block-device registration. + +`sdmmc_protocol::sdio::SdioSdmmc::new_host2` drives reset, power, voltage, +bus-width, clock, and SD/MMC commands through the native `sdio-host2` +submit/poll model. Install optional DMA capability with `PhytiumMci::set_dma` +before handing the host to the protocol layer; block data transactions then try +IDMAC first for 512-byte CMD17/CMD18/CMD24/CMD25 requests and fall back to FIFO +when DMA is unavailable or not applicable. diff --git a/drivers/blk/phytium-mci-host/src/command.rs b/drivers/blk/phytium-mci-host/src/command.rs index 2d8b74da50..c2f8d9101b 100644 --- a/drivers/blk/phytium-mci-host/src/command.rs +++ b/drivers/blk/phytium-mci-host/src/command.rs @@ -1,7 +1,7 @@ use sdmmc_protocol::{ CommandPoll, CommandResponsePoll, cmd::{Command as ProtoCmd, DataDirection}, - error::{Error, Phase}, + error::{Error, ErrorContext, Phase}, response::{ IfCondResponse, OcrResponse, R1Response, RcaResponse, Response, ResponseType, SdioOcrResponse, SdioRwResponse, @@ -19,12 +19,15 @@ pub(crate) enum CommandState { WaitingInhibit { cmd: ProtoCmd, data: Option, + polls: u32, }, WaitingStart { cmd: ProtoCmd, + polls: u32, }, Issued { cmd: ProtoCmd, + polls: u32, }, Complete { response: Response, @@ -56,7 +59,12 @@ impl PhytiumMci { return Err(Error::UnsupportedCommand); } let data = self.pending_data.take(); - self.command_state = CommandState::WaitingInhibit { cmd: *cmd, data }; + self.prepare_irq_for_request(); + self.command_state = CommandState::WaitingInhibit { + cmd: *cmd, + data, + polls: 0, + }; if let Err(err) = self.poll_command() { self.command_state = CommandState::Idle; return Err(err); @@ -66,18 +74,39 @@ impl PhytiumMci { pub fn poll_command(&mut self) -> Result { match self.command_state { - CommandState::WaitingInhibit { cmd, data } => { + CommandState::WaitingInhibit { cmd, data, polls } => { if !self.command_can_issue(data.is_some()) { + if polls >= COMMAND_WAIT_POLLS { + let err = + Error::Timeout(ErrorContext::for_cmd(Phase::CommandSend, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::WaitingInhibit { + cmd, + data, + polls: polls + 1, + }; return Ok(CommandPoll::Pending); } self.program_command(&cmd, data); return Ok(CommandPoll::Pending); } - CommandState::WaitingStart { cmd } => { + CommandState::WaitingStart { cmd, polls } => { if self.regs.cmd().read().start_cmd() { + if polls >= COMMAND_WAIT_POLLS { + let err = + Error::Timeout(ErrorContext::for_cmd(Phase::CommandSend, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::WaitingStart { + cmd, + polls: polls + 1, + }; return Ok(CommandPoll::Pending); } - self.command_state = CommandState::Issued { cmd }; + self.command_state = CommandState::Issued { cmd, polls: 0 }; return Ok(CommandPoll::Pending); } CommandState::Issued { .. } => {} @@ -86,18 +115,18 @@ impl PhytiumMci { CommandState::Idle => return Err(Error::InvalidArgument), } - let CommandState::Issued { cmd } = self.command_state else { + let CommandState::Issued { cmd, polls } = self.command_state else { unreachable!(); }; let raw_status = self.take_command_irq_status(); let status = RIntSts::from_bits(raw_status); if status.error() { - let err = self.translate_int_error(status, Phase::ResponseWait, cmd.cmd); + let err = self.translate_int_error(status, Phase::ResponseWait, cmd.index); self.command_state = CommandState::Failed { error: err }; return Err(err); } if status.command_done() { - let response = match decode_response(self, cmd.resp_type) { + let response = match decode_response(self, cmd.response) { Ok(r) => r, Err(err) => { // Park the FSM in Failed before propagating so the next @@ -111,6 +140,15 @@ impl PhytiumMci { self.command_state = CommandState::Complete { response }; return Ok(CommandPoll::Complete); } + if polls >= COMMAND_WAIT_POLLS { + let err = Error::Timeout(ErrorContext::for_cmd(Phase::ResponseWait, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::Issued { + cmd, + polls: polls + 1, + }; Ok(CommandPoll::Pending) } @@ -118,10 +156,14 @@ impl PhytiumMci { match self.command_state { CommandState::Complete { response } => { self.command_state = CommandState::Idle; + if self.data_cmd_index == 0 { + self.irq.state.end_request(); + } Ok(response) } CommandState::Failed { error } => { self.command_state = CommandState::Idle; + self.irq.state.end_request(); Err(error) } CommandState::Idle @@ -139,7 +181,7 @@ impl PhytiumMci { fn program_command(&mut self, cmd: &ProtoCmd, data: Option) { if data.is_some() { - self.data_cmd_index = cmd.cmd; + self.data_cmd_index = cmd.index; } self.clear_command_int_status(); let mut use_idmac = false; @@ -148,24 +190,31 @@ impl PhytiumMci { use_idmac = d.use_idmac; d.direction }); - self.regs.cmdarg().write(cmd.arg); + self.regs.cmdarg().write(cmd.argument); let mut encoded = encode_command(cmd, data_dir).with_use_hold_reg(self.use_hold_reg); if use_idmac { encoded = encoded.with_transfer_mode(true); } self.regs.cmd().write(encoded); - self.command_state = CommandState::WaitingStart { cmd: *cmd }; + self.command_state = CommandState::WaitingStart { + cmd: *cmd, + polls: 0, + }; } fn take_command_irq_status(&mut self) -> u32 { + if self.completion_irq_enabled() { + return self + .irq + .state + .take_status(crate::MCI_INT_COMMAND_DONE | crate::MCI_INT_ERROR_MASK); + } let raw_status = self.regs.rintsts().read().into_bits(); let consume = raw_status & (crate::MCI_INT_COMMAND_DONE | crate::MCI_INT_ERROR_MASK); if consume != 0 { self.regs.rintsts().write(RIntSts::from_bits(consume)); } - self.irq_state - .take_status(crate::MCI_INT_COMMAND_DONE | crate::MCI_INT_ERROR_MASK) - | raw_status + raw_status } fn clear_command_int_status(&mut self) { @@ -174,19 +223,51 @@ impl PhytiumMci { if raw_status != 0 { self.regs.rintsts().write(RIntSts::from_bits(raw_status)); } - self.irq_state + self.irq + .state .clear_status(crate::MCI_INT_COMMAND_DONE | crate::MCI_INT_ERROR_MASK); } + + fn prepare_irq_for_request(&mut self) { + self.clear_all_int_status(); + self.regs.idsts().write(u32::MAX); + self.irq.state.begin_request(); + } + + pub(crate) fn abort_command(&mut self) -> Result<(), Error> { + self.clear_command_int_status(); + for _ in 0..COMMAND_WAIT_POLLS { + if !self.regs.cmd().read().start_cmd() { + self.clear_all_int_status(); + self.reset_fifo(Phase::CommandSend)?; + self.reset_dma(Phase::CommandSend)?; + self.pending_data = None; + self.data_blocks_remaining = 0; + self.data_cmd_index = 0; + self.command_state = CommandState::Idle; + return Ok(()); + } + core::hint::spin_loop(); + } + self.reset_and_init_preserving_irq()?; + self.pending_data = None; + self.data_blocks_remaining = 0; + self.data_cmd_index = 0; + self.command_state = CommandState::Idle; + Ok(()) + } } +const COMMAND_WAIT_POLLS: u32 = 1_000_000; + pub(crate) fn encode_command(cmd: &ProtoCmd, data_dir: Option) -> Cmd { let mut c = Cmd::new() .with_start_cmd(true) .with_use_hold_reg(true) .with_wait_prvdata_complete(true) - .with_cmd_index(cmd.cmd & 0x3F); + .with_cmd_index(cmd.index & 0x3F); - match cmd.resp_type { + match cmd.response { ResponseType::None => {} ResponseType::R1 | ResponseType::R5 | ResponseType::R6 | ResponseType::R7 => { c = c.with_response_expect(true).with_check_response_crc(true); @@ -207,10 +288,10 @@ pub(crate) fn encode_command(cmd: &ProtoCmd, data_dir: Option) -> _ => {} } - if cmd.cmd == 0 { + if cmd.index == 0 { c = c.with_send_initialization(true); } - if cmd.cmd == 12 { + if cmd.index == 12 { c = c.with_stop_abort_cmd(true); } diff --git a/drivers/blk/phytium-mci-host/src/dma.rs b/drivers/blk/phytium-mci-host/src/dma.rs index d43482c041..d031a2642e 100644 --- a/drivers/blk/phytium-mci-host/src/dma.rs +++ b/drivers/blk/phytium-mci-host/src/dma.rs @@ -1,6 +1,9 @@ +use alloc::boxed::Box; use core::{num::NonZeroUsize, ptr::NonNull}; -use dma_api::{CoherentArray, DeviceDma, DmaDirection, StreamingMap}; +use dma_api::{ + CoherentArray, CompletedDma, CpuDmaBuffer, DeviceDma, DmaDirection, InFlightDma, PreparedDma, +}; use log::warn; use sdmmc_protocol::{ block::{ @@ -32,8 +35,11 @@ const IDSTS_RECEIVE: u32 = crate::MCI_IDSTS_RECEIVE; const IDSTS_NORMAL_SUMMARY: u32 = 1 << 8; const IDSTS_ABNORMAL_SUMMARY: u32 = 1 << 9; const IDSTS_ERROR_MASK: u32 = crate::MCI_IDSTS_ERROR_MASK | IDSTS_ABNORMAL_SUMMARY; -const IDSTS_INT_ENABLE_MASK: u32 = - crate::MCI_IDSTS_ERROR_MASK | IDSTS_NORMAL_SUMMARY | IDSTS_ABNORMAL_SUMMARY; +const IDSTS_INT_ENABLE_MASK: u32 = IDSTS_TRANSMIT + | IDSTS_RECEIVE + | crate::MCI_IDSTS_ERROR_MASK + | IDSTS_NORMAL_SUMMARY + | IDSTS_ABNORMAL_SUMMARY; #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -50,7 +56,7 @@ struct IdmacDesc { struct DmaProgress { descriptors: CoherentArray, - buffer: StreamingMap, + buffer: DmaRequestBuffer, desc_count: usize, complete: bool, idmac_done: bool, @@ -62,6 +68,62 @@ impl DmaProgress { let _ = self.descriptors.dma_addr(); let _ = self.desc_count; } + + fn is_done(&self) -> bool { + self.data_done && self.idmac_done + } + + fn complete(self, read: bool) -> Option { + self.buffer.complete(read) + } + + fn abort(self, read: bool, quiesced: bool) -> Option { + self.buffer.finish(read, quiesced) + } +} + +enum DmaRequestBuffer { + Bounce { + buffer: InFlightDma, + readback: Option<(NonNull, usize)>, + }, + Owned(InFlightDma), +} + +impl DmaRequestBuffer { + fn complete(self, read: bool) -> Option { + self.finish(read, true) + } + + fn finish(self, read: bool, quiesced: bool) -> Option { + match self { + Self::Bounce { buffer, readback } => { + if !quiesced { + let _quarantined = buffer.quarantine(); + return None; + } + if read { + let completed = unsafe { buffer.complete_after_quiesce() }; + if let Some((dst, len)) = readback { + completed.copy_from_device_to_slice(unsafe { + core::slice::from_raw_parts_mut(dst.as_ptr(), len) + }); + } + None + } else { + drop(unsafe { buffer.complete_after_quiesce() }); + None + } + } + Self::Owned(in_flight) => { + if !quiesced { + let _quarantined = in_flight.quarantine(); + return None; + } + Some(unsafe { in_flight.complete_after_quiesce() }) + } + } + } } pub type RequestId = BlockRequestId; @@ -70,9 +132,14 @@ pub type RequestId = BlockRequestId; pub struct BlockRequestSlot { next: usize, state: BlockTransferState, + completed_dma: Option, } impl BlockRequestSlot { + pub fn take_completed_dma(&mut self) -> Option { + self.completed_dma.take() + } + pub fn start( &mut self, mode: BlockTransferMode, @@ -92,10 +159,19 @@ impl BlockRequestSlot { } pub fn complete(&mut self, id: RequestId) -> Result<(), Error> { + self.complete_with_dma(id, None) + } + + fn complete_with_dma( + &mut self, + id: RequestId, + completed_dma: Option, + ) -> Result<(), Error> { if self.state.id() != Some(id) { return Err(Error::InvalidArgument); } self.state = BlockTransferState::Idle; + self.completed_dma = completed_dma; Ok(()) } @@ -108,6 +184,24 @@ pub struct BlockRequest { inner: BlockRequestKind, } +pub struct PreparedDmaSubmitError { + pub error: Error, + buffer: Box, +} + +impl PreparedDmaSubmitError { + fn new(error: Error, buffer: PreparedDma) -> Self { + Self { + error, + buffer: Box::new(buffer), + } + } + + pub fn into_buffer(self) -> PreparedDma { + *self.buffer + } +} + unsafe impl Send for BlockRequest {} enum BlockRequestKind { @@ -211,6 +305,14 @@ impl BlockRequest { | BlockRequestKind::DmaWrite { response, .. } => *response, } } + + fn dma_progress_done(&self) -> bool { + match &self.inner { + BlockRequestKind::DmaRead { progress, .. } + | BlockRequestKind::DmaWrite { progress, .. } => progress.is_done(), + BlockRequestKind::FifoRead { .. } | BlockRequestKind::FifoWrite { .. } => true, + } + } } impl PhytiumMci { @@ -223,13 +325,14 @@ impl PhytiumMci { mode: BlockTransferMode, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let id = slot.start(mode, BlockTransferDirection::Read)?; let result = match mode { BlockTransferMode::Dma => self.build_dma_read_request( start_block, buffer, size, - dma.ok_or(Error::InvalidArgument)?, + dma.ok_or(Error::UnsupportedCommand)?, id, ), BlockTransferMode::Fifo => self.build_fifo_read_request(start_block, buffer, size, id), @@ -254,13 +357,14 @@ impl PhytiumMci { mode: BlockTransferMode, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let id = slot.start(mode, BlockTransferDirection::Write)?; let result = match mode { BlockTransferMode::Dma => self.build_dma_write_request( start_block, buffer, size, - dma.ok_or(Error::InvalidArgument)?, + dma.ok_or(Error::UnsupportedCommand)?, id, ), BlockTransferMode::Fifo => self.build_fifo_write_request(start_block, buffer, size, id), @@ -276,6 +380,52 @@ impl PhytiumMci { } } + pub fn submit_prepared_read_blocks( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + slot: &mut BlockRequestSlot, + ) -> Result { + if let Err(err) = self.check_not_poisoned() { + return Err(PreparedDmaSubmitError::new(err, buffer)); + } + let id = match slot.start(BlockTransferMode::Dma, BlockTransferDirection::Read) { + Ok(id) => id, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + match self.build_prepared_dma_read_request(start_block, buffer, dma, id) { + Ok(request) => Ok(request), + Err(err) => { + let _ = slot.complete(id); + Err(err) + } + } + } + + pub fn submit_prepared_write_blocks( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + slot: &mut BlockRequestSlot, + ) -> Result { + if let Err(err) = self.check_not_poisoned() { + return Err(PreparedDmaSubmitError::new(err, buffer)); + } + let id = match slot.start(BlockTransferMode::Dma, BlockTransferDirection::Write) { + Ok(id) => id, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + match self.build_prepared_dma_write_request(start_block, buffer, dma, id) { + Ok(request) => Ok(request), + Err(err) => { + let _ = slot.complete(id); + Err(err) + } + } + } + pub fn poll_block_request( &mut self, request: &mut Option, @@ -305,6 +455,15 @@ impl PhytiumMci { self.poll_data_request_inner(request, id, slot) } + pub fn abort_block_request_response( + &mut self, + request: &mut Option, + id: RequestId, + slot: &mut BlockRequestSlot, + ) -> Result<(), Error> { + self.abort_block_request(request, id, slot, Phase::DataRead) + } + fn build_fifo_read_request( &mut self, start_block: u32, @@ -438,27 +597,35 @@ impl PhytiumMci { DataDirection::None => return Err(Error::InvalidArgument), _ => return Err(Error::InvalidArgument), }; - let slice = unsafe { core::slice::from_raw_parts_mut(buffer.as_ptr(), len) }; - let mapped = dma - .map_streaming_slice_for_device(slice, BLOCK_SIZE, dma_direction) - .map_err(|_| Error::Misaligned)?; + let mut backing = CpuDmaBuffer::new_zero( + dma, + NonZeroUsize::new(len).ok_or(Error::InvalidArgument)?, + block_size_usize, + dma_direction, + ) + .map_err(|_| Error::Misaligned)?; + if direction == DataDirection::Write { + backing.copy_to_device_from_slice(unsafe { + core::slice::from_raw_parts(buffer.as_ptr(), len) + }); + } + let dma_addr = backing.dma_addr().as_u64(); + let in_flight = unsafe { backing.prepare_for_device().into_in_flight() }; let desc_count = len.div_ceil(IDMAC_MAX_BUF_SIZE); let mut descriptors = dma .coherent_array_zero_with_align::(desc_count, IDMAC_DESC_ALIGN) .map_err(|_| Error::Misaligned)?; let desc_dma = descriptors.dma_addr().as_u64(); - let desc_values = build_idmac_descriptors( - mapped.dma_addr().as_u64(), - desc_dma, - len, - IDMAC_MAX_BUF_SIZE, - )?; + let desc_values = build_idmac_descriptors(dma_addr, desc_dma, len, IDMAC_MAX_BUF_SIZE)?; descriptors.write_with_cpu(desc_values.len(), |dst| dst.copy_from_slice(&desc_values)); self.start_idmac_transfer(cmd, block_size, block_count, desc_dma)?; let progress = DmaProgress { descriptors, - buffer: mapped, + buffer: DmaRequestBuffer::Bounce { + buffer: in_flight, + readback: (direction == DataDirection::Read).then_some((buffer, len)), + }, desc_count, complete: false, idmac_done: false, @@ -468,7 +635,7 @@ impl PhytiumMci { DataDirection::Read => BlockRequestKind::DmaRead { id, progress, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase, stage: BlockRequestStage::Command, stop_after_complete, @@ -477,7 +644,7 @@ impl PhytiumMci { DataDirection::Write => BlockRequestKind::DmaWrite { id, progress, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase, stage: BlockRequestStage::Command, stop_after_complete, @@ -489,6 +656,155 @@ impl PhytiumMci { Ok(BlockRequest { inner }) } + fn build_prepared_dma_read_request( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + id: RequestId, + ) -> Result { + if buffer.direction() != DmaDirection::FromDevice || buffer.domain_id() != dma.domain_id() { + return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)); + } + let block_count = match block_count(buffer.len()) { + Ok(block_count) => block_count, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + let cmd = if block_count == 1 { + cmd17(start_block) + } else { + cmd18(start_block) + }; + self.build_prepared_dma_data_request( + &cmd, + buffer, + BLOCK_SIZE as u32, + block_count, + dma, + id, + DataDirection::Read, + block_count > 1, + ) + } + + fn build_prepared_dma_write_request( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + id: RequestId, + ) -> Result { + if buffer.direction() != DmaDirection::ToDevice || buffer.domain_id() != dma.domain_id() { + return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)); + } + let block_count = match block_count(buffer.len()) { + Ok(block_count) => block_count, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + let cmd = if block_count == 1 { + cmd24(start_block) + } else { + cmd25(start_block) + }; + self.build_prepared_dma_data_request( + &cmd, + buffer, + BLOCK_SIZE as u32, + block_count, + dma, + id, + DataDirection::Write, + block_count > 1, + ) + } + + #[allow(clippy::too_many_arguments)] + fn build_prepared_dma_data_request( + &mut self, + cmd: &Command, + buffer: PreparedDma, + block_size: u32, + block_count: u32, + dma: &DeviceDma, + id: RequestId, + direction: DataDirection, + stop_after_complete: bool, + ) -> Result { + let block_size_usize = match usize::try_from(block_size) { + Ok(block_size) => block_size, + Err(_) => return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)), + }; + if block_size_usize == 0 + || buffer.len().get() != block_size_usize.saturating_mul(block_count as usize) + { + return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)); + } + let phase = match direction { + DataDirection::Read => Phase::DataRead, + DataDirection::Write => Phase::DataWrite, + DataDirection::None => { + return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)); + } + _ => return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)), + }; + let len = buffer.len().get(); + let desc_count = len.div_ceil(IDMAC_MAX_BUF_SIZE); + let mut descriptors = + match dma.coherent_array_zero_with_align::(desc_count, IDMAC_DESC_ALIGN) { + Ok(descriptors) => descriptors, + Err(_) => return Err(PreparedDmaSubmitError::new(Error::Misaligned, buffer)), + }; + let desc_dma = descriptors.dma_addr().as_u64(); + let desc_values = match build_idmac_descriptors( + buffer.dma_addr().as_u64(), + desc_dma, + len, + IDMAC_MAX_BUF_SIZE, + ) { + Ok(desc_values) => desc_values, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + descriptors.write_with_cpu(desc_values.len(), |dst| dst.copy_from_slice(&desc_values)); + match self.start_idmac_transfer(cmd, block_size, block_count, desc_dma) { + Ok(()) => {} + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + } + + let progress = DmaProgress { + descriptors, + buffer: DmaRequestBuffer::Owned(unsafe { buffer.into_in_flight() }), + desc_count, + complete: false, + idmac_done: false, + data_done: false, + }; + let inner = match direction { + DataDirection::Read => BlockRequestKind::DmaRead { + id, + progress, + cmd_index: cmd.index, + phase, + stage: BlockRequestStage::Command, + stop_after_complete, + response: None, + }, + DataDirection::Write => BlockRequestKind::DmaWrite { + id, + progress, + cmd_index: cmd.index, + phase, + stage: BlockRequestStage::Command, + stop_after_complete, + response: None, + }, + DataDirection::None => { + unreachable!("DataDirection::None returned before DMA request construction") + } + _ => unreachable!("unsupported DataDirection returned before DMA request construction"), + }; + Ok(BlockRequest { inner }) + } + #[allow(clippy::too_many_arguments)] pub fn submit_fifo_data_request( &mut self, @@ -500,6 +816,7 @@ impl PhytiumMci { direction: DataDirection, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let transfer_direction = match direction { DataDirection::Read => BlockTransferDirection::Read, DataDirection::Write => BlockTransferDirection::Write, @@ -568,7 +885,7 @@ impl PhytiumMci { len, block_size: block_size_usize, progress: FifoProgress::default(), - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase, stage: BlockRequestStage::Command, stop_after_complete, @@ -580,7 +897,7 @@ impl PhytiumMci { len, block_size: block_size_usize, progress: FifoProgress::default(), - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase, stage: BlockRequestStage::Command, stop_after_complete, @@ -644,7 +961,7 @@ impl PhytiumMci { // Future CommandPoll variants: best-effort, treat as still pending. Ok(_) => return Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); return Err(err); } } @@ -661,7 +978,7 @@ impl PhytiumMci { // Future BlockPoll variants: best-effort, treat as still pending. Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); Err(err) } } @@ -700,7 +1017,7 @@ impl PhytiumMci { } Ok(_) => return Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); return Err(err); } } @@ -716,7 +1033,7 @@ impl PhytiumMci { Ok(BlockPoll::Complete) => self.finish_dma_data(request, id, slot), Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); Err(err) } } @@ -733,9 +1050,9 @@ impl PhytiumMci { let Some(active) = request.as_mut() else { return Err(Error::InvalidArgument); }; - let (progress, is_read) = match &mut active.inner { - BlockRequestKind::DmaRead { progress, .. } => (progress, true), - BlockRequestKind::DmaWrite { progress, .. } => (progress, false), + let progress = match &mut active.inner { + BlockRequestKind::DmaRead { progress, .. } => progress, + BlockRequestKind::DmaWrite { progress, .. } => progress, _ => return Err(Error::InvalidArgument), }; @@ -758,13 +1075,10 @@ impl PhytiumMci { } progress.idmac_done |= raw_idsts & (IDSTS_RECEIVE | IDSTS_TRANSMIT) != 0; progress.data_done |= ints.data_transfer_over(); - if !progress.data_done { + if !progress.is_done() { return Ok(BlockPoll::Pending); } - if is_read { - progress.buffer.complete_for_cpu_all(); - } progress.complete = true; Ok(BlockPoll::Complete) } @@ -775,23 +1089,28 @@ impl PhytiumMci { id: RequestId, slot: &mut BlockRequestSlot, ) -> Result { - self.disable_idmac(); let stop_after_complete = match request.as_mut().map(|r| &mut r.inner) { Some(BlockRequestKind::DmaRead { stage, stop_after_complete, + progress, .. }) | Some(BlockRequestKind::DmaWrite { stage, stop_after_complete, + progress, .. }) => { + if !progress.is_done() { + return Ok(DataCommandPoll::Pending); + } *stage = BlockRequestStage::Stop; *stop_after_complete } _ => return Err(Error::InvalidArgument), }; + self.disable_idmac(); if stop_after_complete { self.submit_command(&CMD12)?; return Ok(DataCommandPoll::Pending); @@ -799,8 +1118,8 @@ impl PhytiumMci { let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active); - slot.complete(id)?; + let completed_dma = self.finish_block_request(active); + slot.complete_with_dma(id, completed_dma)?; Ok(DataCommandPoll::Complete(response)) } @@ -861,7 +1180,8 @@ impl PhytiumMci { let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active); + let completed_dma = self.finish_block_request(active); + drop(completed_dma); slot.complete(id)?; Ok(DataCommandPoll::Complete(response)) } @@ -877,30 +1197,69 @@ impl PhytiumMci { Ok(CommandPoll::Pending) => Ok(DataCommandPoll::Pending), Ok(CommandPoll::Complete) => { let _ = self.take_command_response()?; + if !request + .as_ref() + .is_some_and(|active| active.dma_progress_done()) + { + return Ok(DataCommandPoll::Pending); + } let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active); - slot.complete(id)?; + let completed_dma = self.finish_block_request(active); + slot.complete_with_dma(id, completed_dma)?; Ok(DataCommandPoll::Complete(response)) } // Future CommandPoll variants: best-effort, treat as still pending. Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot, phase); + let _ = self.abort_block_request(request, id, slot, phase); Err(err) } } } - fn finish_block_request(&mut self, request: BlockRequest) { - match &request.inner { - BlockRequestKind::DmaRead { progress, .. } - | BlockRequestKind::DmaWrite { progress, .. } => progress.keep_alive(), - _ => {} + fn finish_block_request(&mut self, request: BlockRequest) -> Option { + self.finish_block_request_with_quiesce(request, true) + } + + fn finish_block_request_with_quiesce( + &mut self, + request: BlockRequest, + quiesced: bool, + ) -> Option { + if !quiesced { + self.poison_dma(); + core::mem::forget(request); + self.pending_data = None; + self.data_blocks_remaining = 0; + self.data_cmd_index = 0; + self.irq.state.end_request(); + return None; } + let completed_dma = match request.inner { + BlockRequestKind::DmaRead { progress, .. } => { + progress.keep_alive(); + if quiesced { + progress.complete(true) + } else { + progress.abort(true, false) + } + } + BlockRequestKind::DmaWrite { progress, .. } => { + progress.keep_alive(); + if quiesced { + progress.complete(false) + } else { + progress.abort(false, false) + } + } + BlockRequestKind::FifoRead { .. } | BlockRequestKind::FifoWrite { .. } => None, + }; self.pending_data = None; self.data_blocks_remaining = 0; self.data_cmd_index = 0; + self.irq.state.end_request(); + completed_dma } fn abort_block_request( @@ -909,13 +1268,34 @@ impl PhytiumMci { id: RequestId, slot: &mut BlockRequestSlot, phase: Phase, - ) { - if let Some(active) = request.take() { - self.finish_block_request(active); - } + ) -> Result<(), Error> { + let active = request.take().ok_or(Error::InvalidArgument)?; self.disable_idmac(); - let _ = self.reset_fifo(phase); - let _ = slot.complete(id); + let fifo = self.reset_fifo(phase); + let dma = self.reset_dma(phase); + self.clear_all_int_status(); + self.command_state = crate::command::CommandState::Idle; + let recovery = match (fifo, dma) { + (Ok(()), Ok(())) => Ok(()), + (Err(err), _) | (_, Err(err)) => { + let reset = self.reset_and_init_preserving_irq(); + self.disable_idmac(); + match reset { + Ok(()) => { + warn!( + "phytium-mci: recovered IDMAC {:?} error by controller reset: {err:?}", + phase + ); + Ok(()) + } + Err(reset_err) => Err(reset_err), + } + } + }; + let completed_dma = self.finish_block_request_with_quiesce(active, recovery.is_ok()); + drop(completed_dma); + slot.complete(id)?; + recovery } fn start_idmac_transfer( @@ -927,14 +1307,16 @@ impl PhytiumMci { ) -> Result<(), Error> { self.clear_all_int_status(); self.regs.idsts().write(u32::MAX); - self.irq_state.clear_all(); - self.regs.idinten().write(IDSTS_INT_ENABLE_MASK); + self.irq.state.clear_all(); + self.regs.idinten().write(0); self.reset_fifo(Phase::Init)?; self.reset_dma(Phase::Init)?; self.program_data_phase(block_size, block_count); self.program_idmac_registers(desc_dma); + self.regs.idsts().write(u32::MAX); + self.regs.idinten().write(IDSTS_INT_ENABLE_MASK); self.pending_data = Some(PendingData { - direction: if matches!(cmd.cmd, 24 | 25) { + direction: if matches!(cmd.index, 24 | 25) { DataDirection::Write } else { DataDirection::Read @@ -948,16 +1330,17 @@ impl PhytiumMci { } fn program_idmac_registers(&self, desc_dma: u64) { + self.regs.dbaddrl().write(desc_dma as u32); + self.regs.dbaddrh().write((desc_dma >> 32) as u32); self.regs.ctrl().update(|r| { r.with_dma_enable(true) .with_use_internal_dmac(true) - .with_int_enable(true) + .with_int_enable(self.completion_irq_enabled()) }); self.regs .bmod() .write(self.regs.bmod().read() | BMOD_FIXED_BURST | BMOD_IDMAC_ENABLE); - self.regs.dbaddrl().write(desc_dma as u32); - self.regs.dbaddrh().write((desc_dma >> 32) as u32); + self.regs.pldmnd().write(1); } fn disable_idmac(&mut self) { @@ -969,31 +1352,32 @@ impl PhytiumMci { } fn take_idmac_status(&mut self) -> u32 { + let mask = IDSTS_RECEIVE | IDSTS_TRANSMIT | IDSTS_ERROR_MASK; + if self.completion_irq_enabled() { + return self.irq.state.take_idmac_status(mask); + } let raw = self.regs.idsts().read(); if raw != 0 { self.regs.idsts().write(raw); } - self.irq_state - .take_idmac_status(IDSTS_RECEIVE | IDSTS_TRANSMIT | IDSTS_ERROR_MASK) - | raw + raw } fn take_data_irq_status(&mut self, cmd_index: u8, phase: Phase) -> Result { - let raw_status = self.regs.rintsts().read().into_bits(); - let consume = raw_status - & (crate::MCI_INT_DATA_TRANSFER_OVER - | crate::MCI_INT_RXDR - | crate::MCI_INT_TXDR - | crate::MCI_INT_ERROR_MASK); - if consume != 0 { - self.regs.rintsts().write(RIntSts::from_bits(consume)); - } - let status = self.irq_state.take_status( - crate::MCI_INT_DATA_TRANSFER_OVER - | crate::MCI_INT_RXDR - | crate::MCI_INT_TXDR - | crate::MCI_INT_ERROR_MASK, - ) | raw_status; + let mask = crate::MCI_INT_DATA_TRANSFER_OVER + | crate::MCI_INT_RXDR + | crate::MCI_INT_TXDR + | crate::MCI_INT_ERROR_MASK; + let status = if self.completion_irq_enabled() { + self.irq.state.take_status(mask) + } else { + let raw_status = self.regs.rintsts().read().into_bits(); + let consume = raw_status & mask; + if consume != 0 { + self.regs.rintsts().write(RIntSts::from_bits(consume)); + } + raw_status + }; let ints = RIntSts::from_bits(status); if ints.error() { return Err(self.translate_int_error(ints, phase, cmd_index)); @@ -1220,6 +1604,13 @@ mod tests { assert_eq!(descriptors[2].desc_lo, 0); } + #[test] + fn idmac_interrupt_mask_enables_terminal_status_bits() { + assert_ne!(IDSTS_INT_ENABLE_MASK & IDSTS_RECEIVE, 0); + assert_ne!(IDSTS_INT_ENABLE_MASK & IDSTS_TRANSMIT, 0); + assert_ne!(IDSTS_INT_ENABLE_MASK & IDSTS_NORMAL_SUMMARY, 0); + } + use core::ptr::NonNull; use ::alloc::{alloc, boxed::Box}; @@ -1234,14 +1625,22 @@ mod tests { impl NoopDmaBuffer { fn progress() -> DmaProgress { - let dma = DeviceDma::new(u64::MAX, &TEST_DMA); + let dma = DeviceDma::new_legacy(u64::MAX, &TEST_DMA); let descriptors = dma .coherent_array_zero_with_align::(1, IDMAC_DESC_ALIGN) .unwrap(); + let buffer = CpuDmaBuffer::new_zero( + &dma, + NonZeroUsize::new(BLOCK_SIZE).unwrap(), + BLOCK_SIZE, + DmaDirection::FromDevice, + ) + .unwrap() + .prepare_for_device(); + let buffer = unsafe { buffer.into_in_flight() }; let backing = Box::leak(Box::new(AlignedBlock([0u8; BLOCK_SIZE]))); - let buffer = dma - .map_streaming_slice(&mut backing.0, BLOCK_SIZE, DmaDirection::FromDevice) - .unwrap(); + let readback = Some((NonNull::from(&mut backing.0[0]), BLOCK_SIZE)); + let buffer = DmaRequestBuffer::Bounce { buffer, readback }; DmaProgress { descriptors, buffer, @@ -1338,14 +1737,14 @@ mod tests { let ctrl = crate::regs::Ctrl::from_bits(mmio[CTRL_WORD]); assert!(ctrl.dma_enable()); assert!(ctrl.use_internal_dmac()); - assert!(ctrl.int_enable()); - assert_eq!(mmio[PLDMND_WORD], 0); + assert!(!ctrl.int_enable()); + assert_eq!(mmio[PLDMND_WORD], 1); assert_eq!(mmio[DBADDRL_WORD], 0x8000_0000); assert_eq!(mmio[DBADDRL_WORD + 1], 1); } #[test] - fn idmac_read_completes_when_data_done_arrives_without_idmac_done() { + fn idmac_read_waits_when_data_done_arrives_without_idmac_done() { let mut mmio = [0u32; 256]; let mut host = host_from_words(&mut mmio); let mut request = Some(BlockRequest { @@ -1369,7 +1768,7 @@ mod tests { assert_eq!( host.poll_dma_data_step(&mut request, 17, Phase::DataRead) .unwrap(), - BlockPoll::Complete + BlockPoll::Pending ); } @@ -1412,6 +1811,30 @@ mod tests { ); } + #[test] + fn request_slot_returns_completed_owned_dma_once() { + let dma = DeviceDma::new_legacy(u64::MAX, &TEST_DMA); + let buffer = dma_api::CpuDmaBuffer::new_zero( + &dma, + NonZeroUsize::new(BLOCK_SIZE).unwrap(), + BLOCK_SIZE, + DmaDirection::FromDevice, + ) + .unwrap() + .prepare_for_device(); + let in_flight = unsafe { buffer.into_in_flight() }; + let completed = DmaRequestBuffer::Owned(in_flight).complete(true).unwrap(); + let mut slot = BlockRequestSlot::default(); + let id = slot + .start(BlockTransferMode::Dma, BlockTransferDirection::Read) + .unwrap(); + + slot.complete_with_dma(id, Some(completed)).unwrap(); + + assert!(slot.take_completed_dma().is_some()); + assert!(slot.take_completed_dma().is_none()); + } + #[test] fn fifo_read_completes_when_dto_arrives_before_fifo_is_drained() { let mut mmio = [0u32; 256]; diff --git a/drivers/blk/phytium-mci-host/src/host.rs b/drivers/blk/phytium-mci-host/src/host.rs index 50ab6e6e0b..2211437801 100644 --- a/drivers/blk/phytium-mci-host/src/host.rs +++ b/drivers/blk/phytium-mci-host/src/host.rs @@ -1,8 +1,10 @@ +use alloc::sync::Arc; use core::{ ptr::NonNull, - sync::atomic::{self, AtomicBool, AtomicU32, Ordering}, + sync::atomic::{self, AtomicBool, AtomicU32, AtomicU64, Ordering}, }; +use dma_api::DeviceDma; use mmio_api::MmioRaw; use sdmmc_protocol::{ error::{Error, ErrorContext, Phase}, @@ -22,9 +24,9 @@ use crate::{ pub const DEFAULT_FIFO_OFFSET: usize = 0x200; const DEFAULT_FIFO_WORD_DEPTH: u32 = 128; -const FIFO_THRESHOLD: u32 = (2 << 28) | (7 << 16) | 0x100; -const CARD_READ_THRESHOLD_ENABLE: u32 = 1; -const CARD_READ_THRESHOLD_DEPTH8: u32 = 1 << 23; +pub(crate) const FIFO_THRESHOLD: u32 = (2 << 28) | (7 << 16) | 0x100; +pub(crate) const CARD_READ_THRESHOLD_ENABLE: u32 = 1; +pub(crate) const CARD_READ_THRESHOLD_DEPTH8: u32 = 1 << 23; const BMOD_SOFTWARE_RESET: u32 = 1; const RESET_POLL_LIMIT: usize = 1_000_000; const CLOCK_POLL_LIMIT: usize = 1_000_000; @@ -38,68 +40,167 @@ pub(crate) struct PendingData { } pub(crate) struct IrqState { - pending_status: AtomicU32, - pending_idmac_status: AtomicU32, + status_mailbox: AtomicU64, + idmac_mailbox: AtomicU64, + next_generation: AtomicU32, } +const IRQ_GENERATION_SHIFT: u64 = 32; +const IRQ_STATUS_MASK: u64 = u32::MAX as u64; + impl IrqState { const fn new() -> Self { Self { - pending_status: AtomicU32::new(0), - pending_idmac_status: AtomicU32::new(0), + status_mailbox: AtomicU64::new(0), + idmac_mailbox: AtomicU64::new(0), + next_generation: AtomicU32::new(0), } } - pub(crate) fn cache_status(&self, status: u32) { - if status != 0 { - self.pending_status.fetch_or(status, Ordering::AcqRel); - } + pub(crate) fn begin_request(&self) { + let generation = self.next_generation(); + let clean = pack_mailbox(generation, 0); + self.idmac_mailbox.store(clean, Ordering::Release); + self.status_mailbox.store(clean, Ordering::Release); } - pub(crate) fn cache_idmac_status(&self, status: u32) { + pub(crate) fn end_request(&self) { + self.status_mailbox.store(0, Ordering::Release); + self.idmac_mailbox.store(0, Ordering::Release); + } + + pub(crate) fn cache_if_current(&self, generation: u32, status: u32, idmac_status: u32) { + if generation == 0 { + return; + } if status != 0 { - self.pending_idmac_status.fetch_or(status, Ordering::AcqRel); + cache_mailbox_if_current(&self.status_mailbox, generation, status); + } + if idmac_status != 0 { + cache_mailbox_if_current(&self.idmac_mailbox, generation, idmac_status); } } + pub(crate) fn generation(&self) -> u32 { + mailbox_generation(self.status_mailbox.load(Ordering::Acquire)) + } + pub(crate) fn take_status(&self, mask: u32) -> u32 { - take_cached_bits(&self.pending_status, mask) + take_mailbox_bits(&self.status_mailbox, mask) } pub(crate) fn take_idmac_status(&self, mask: u32) -> u32 { - take_cached_bits(&self.pending_idmac_status, mask) + take_mailbox_bits(&self.idmac_mailbox, mask) } pub(crate) fn clear_status(&self, mask: u32) { - self.pending_status.fetch_and(!mask, Ordering::AcqRel); + clear_mailbox_bits(&self.status_mailbox, mask); } pub(crate) fn clear_all(&self) { - self.pending_status.store(0, Ordering::Release); - self.pending_idmac_status.store(0, Ordering::Release); + clear_mailbox_bits(&self.status_mailbox, u32::MAX); + clear_mailbox_bits(&self.idmac_mailbox, u32::MAX); } #[cfg(test)] pub(crate) fn pending_status(&self) -> u32 { - self.pending_status.load(Ordering::Acquire) + mailbox_status(self.status_mailbox.load(Ordering::Acquire)) } #[cfg(test)] pub(crate) fn pending_idmac_status(&self) -> u32 { - self.pending_idmac_status.load(Ordering::Acquire) + mailbox_status(self.idmac_mailbox.load(Ordering::Acquire)) + } + + fn next_generation(&self) -> u32 { + let mut cur = self.next_generation.load(Ordering::Acquire); + loop { + let mut next = cur.wrapping_add(1); + if next == 0 { + next = 1; + } + match self.next_generation.compare_exchange_weak( + cur, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return next, + Err(observed) => cur = observed, + } + } } } -fn take_cached_bits(cache: &AtomicU32, mask: u32) -> u32 { - let mut cur = cache.load(Ordering::Acquire); +fn pack_mailbox(generation: u32, status: u32) -> u64 { + ((generation as u64) << IRQ_GENERATION_SHIFT) | status as u64 +} + +fn mailbox_generation(value: u64) -> u32 { + (value >> IRQ_GENERATION_SHIFT) as u32 +} + +fn mailbox_status(value: u64) -> u32 { + (value & IRQ_STATUS_MASK) as u32 +} + +fn cache_mailbox_if_current(mailbox: &AtomicU64, generation: u32, status: u32) { + let mut cur = mailbox.load(Ordering::Acquire); + loop { + if mailbox_generation(cur) != generation { + return; + } + let next = pack_mailbox(generation, mailbox_status(cur) | status); + match mailbox.compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return, + Err(observed) => cur = observed, + } + } +} + +fn take_mailbox_bits(mailbox: &AtomicU64, mask: u32) -> u32 { + let mut cur = mailbox.load(Ordering::Acquire); loop { - let taken = cur & mask; + let status = mailbox_status(cur); + let taken = status & mask; if taken == 0 { return 0; } - match cache.compare_exchange_weak(cur, cur & !mask, Ordering::AcqRel, Ordering::Acquire) { + let next = pack_mailbox(mailbox_generation(cur), status & !mask); + match mailbox.compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) { Ok(_) => return taken, - Err(next) => cur = next, + Err(observed) => cur = observed, + } + } +} + +fn clear_mailbox_bits(mailbox: &AtomicU64, mask: u32) { + let mut cur = mailbox.load(Ordering::Acquire); + loop { + let next = pack_mailbox(mailbox_generation(cur), mailbox_status(cur) & !mask); + match mailbox.compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return, + Err(observed) => cur = observed, + } + } +} + +pub(crate) struct IrqCore { + pub(crate) regs: VolatilePtr<'static, RegisterBlock>, + pub(crate) state: IrqState, +} + +// SAFETY: `IrqCore` is shared only between task-side polling and the IRQ +// top-half. MMIO accesses are volatile and event sharing goes through atomics. +unsafe impl Send for IrqCore {} +// SAFETY: See the `Send` impl. +unsafe impl Sync for IrqCore {} + +impl IrqCore { + fn new(regs: VolatilePtr<'static, RegisterBlock>) -> Self { + Self { + regs, + state: IrqState::new(), } } } @@ -112,9 +213,14 @@ pub struct PhytiumMci { pub(crate) pending_data: Option, pub(crate) data_cmd_index: u8, pub(crate) data_blocks_remaining: u32, + pub(crate) dma: Option, + pub(crate) dma_mask: u64, + pub(crate) dma_poisoned: bool, pub(crate) use_hold_reg: bool, - pub(crate) irq_state: IrqState, + pub(crate) irq: Arc, completion_irq_enabled: AtomicBool, + pub(crate) host2_next_id: u64, + pub(crate) host2_active_id: Option, } impl PhytiumMci { @@ -132,9 +238,14 @@ impl PhytiumMci { pending_data: None, data_cmd_index: 0, data_blocks_remaining: 0, + dma: None, + dma_mask: u32::MAX as u64, + dma_poisoned: false, use_hold_reg: true, - irq_state: IrqState::new(), + irq: Arc::new(IrqCore::new(regs)), completion_irq_enabled: AtomicBool::new(false), + host2_next_id: 0, + host2_active_id: None, } } @@ -147,6 +258,28 @@ impl PhytiumMci { unsafe { Self::new(base) } } + /// Install a DMA capability used by high-level data-transfer hooks. + /// + /// Once installed, `SdioHost` and `sdio_host2::SdioHost` data transactions + /// try the internal IDMAC first for 512-byte block I/O and fall back to the + /// FIFO state machine when the DMA path is not applicable. + pub fn set_dma(&mut self, dma: DeviceDma) { + self.dma_mask = dma.dma_mask(); + self.dma = Some(dma); + } + + pub(crate) fn check_not_poisoned(&self) -> Result<(), Error> { + if self.dma_poisoned { + Err(Error::BusError(ErrorContext::new(Phase::DataRead))) + } else { + Ok(()) + } + } + + pub(crate) fn poison_dma(&mut self) { + self.dma_poisoned = true; + } + pub fn reset_and_init(&mut self) -> Result<(), Error> { self.regs.clkena().write(ClkEna::new()); self.regs.ctrl().update(|r| { @@ -165,7 +298,7 @@ impl PhytiumMci { self.regs.idinten().write(0); self.clear_all_int_status(); self.regs.idsts().write(u32::MAX); - self.irq_state.clear_all(); + self.irq.state.clear_all(); self.completion_irq_enabled.store(false, Ordering::Release); self.regs.ctype().write(CType::new()); @@ -181,6 +314,16 @@ impl PhytiumMci { self.program_timing(TimingTable::sd_for_speed( sdmmc_protocol::sdio::ClockSpeed::Identification, )?)?; + self.dma_poisoned = false; + Ok(()) + } + + pub(crate) fn reset_and_init_preserving_irq(&mut self) -> Result<(), Error> { + let was_irq_enabled = self.completion_irq_enabled(); + self.reset_and_init()?; + if was_irq_enabled { + self.enable_completion_irq(); + } Ok(()) } @@ -269,14 +412,17 @@ impl PhytiumMci { self.regs.ctrl().update(|r| r.with_int_enable(false)); } + pub(crate) fn clear_completion_irq_enabled(&self) { + self.completion_irq_enabled.store(false, Ordering::Release); + } + pub fn completion_irq_enabled(&self) -> bool { self.completion_irq_enabled.load(Ordering::Acquire) } pub fn irq_handle(&self) -> PhytiumMciIrqHandle { PhytiumMciIrqHandle { - regs: self.regs, - irq_state: &self.irq_state, + irq: self.irq.clone(), } } @@ -388,7 +534,7 @@ impl PhytiumMci { (self.base_addr + self.fifo_offset) as *mut u32 } - fn write_ext_reg(&self, offset: usize, value: u32) { + pub(crate) fn write_ext_reg(&self, offset: usize, value: u32) { let ptr = (self.base_addr + offset) as *mut u32; unsafe { ptr.write_volatile(value); @@ -411,16 +557,16 @@ impl sdmmc_protocol::sdio::SdioIrqHandle for PhytiumMciIrqHandle { type Event = Event; fn handle_irq(&self) -> Self::Event { - let raw = self.regs.rintsts().read().into_bits(); - let idsts = self.regs.idsts().read(); + let generation = self.irq.state.generation(); + let raw = self.irq.regs.rintsts().read().into_bits(); + let idsts = self.irq.regs.idsts().read(); if raw != 0 { - self.regs.rintsts().write(RIntSts::from_bits(raw)); - unsafe { &*self.irq_state }.cache_status(raw); + self.irq.regs.rintsts().write(RIntSts::from_bits(raw)); } if idsts != 0 { - self.regs.idsts().write(idsts); - unsafe { &*self.irq_state }.cache_idmac_status(idsts); + self.irq.regs.idsts().write(idsts); } + self.irq.state.cache_if_current(generation, raw, idsts); PhytiumMci::event_from_raw_irq(raw, idsts) } @@ -467,6 +613,8 @@ mod tests { let mut mmio = [0u32; 256]; let base = NonNull::new(mmio.as_mut_ptr().cast()).unwrap(); let host = unsafe { PhytiumMci::new(base) }; + host.irq.state.begin_request(); + let old_generation = host.irq.state.generation(); const IDSTS_WORD: usize = 36; const IDSTS_RECEIVE: u32 = 1 << 1; @@ -477,7 +625,16 @@ mod tests { }; assert_eq!(host.handle_irq(), crate::Event::TransferComplete); - assert_eq!(host.irq_state.pending_idmac_status(), IDSTS_RECEIVE); - assert_eq!(host.irq_state.pending_status(), 0); + assert_eq!(host.irq.state.pending_idmac_status(), IDSTS_RECEIVE); + assert_eq!(host.irq.state.pending_status(), 0); + + let _ = host.irq.state.take_idmac_status(IDSTS_RECEIVE); + host.irq.state.end_request(); + host.irq.state.begin_request(); + assert_ne!(host.irq.state.generation(), old_generation); + host.irq + .state + .cache_if_current(old_generation, 0, IDSTS_RECEIVE); + assert_eq!(host.irq.state.pending_idmac_status(), 0); } } diff --git a/drivers/blk/phytium-mci-host/src/lib.rs b/drivers/blk/phytium-mci-host/src/lib.rs index 5d20eb1565..1eea546fc6 100644 --- a/drivers/blk/phytium-mci-host/src/lib.rs +++ b/drivers/blk/phytium-mci-host/src/lib.rs @@ -8,39 +8,45 @@ //! //! - **Implemented**: controller/FIFO reset, power and clock setup, Phytium //! timing tables, 1-bit / 4-bit / 8-bit bus selection, command response -//! decoding, FIFO block transfers, and stable IRQ event extraction. +//! decoding, FIFO and IDMAC block transfers, and stable IRQ event extraction. //! - **Out of scope for this crate**: FDT/ACPI probe, MMIO remapping, IRQ //! registration, pad-controller programming, OS sleeps/wakeups, and rdif-block //! registration. //! - **Implemented for block I/O**: IDMAC descriptor setup, DMA buffer mapping, -//! DMA block read/write polling, and FIFO fallback at the platform adapter. +//! DMA block read/write polling, and FIFO fallback in the native protocol +//! data path. #![no_std] #![allow(clippy::missing_safety_doc)] extern crate alloc; -use core::{marker::PhantomData, ptr::NonNull}; +use alloc::sync::Arc; +use core::{marker::PhantomData, num::NonZeroUsize, ptr::NonNull}; mod command; mod dma; mod host; +pub mod rdif; mod regs; mod timing; pub use dma::{BlockRequest, BlockRequestSlot, RequestId}; +use host::uhs_bits_after_voltage; pub use host::{DEFAULT_FIFO_OFFSET, PhytiumMci}; +use regs::RegisterBlockVolatileFieldAccess; pub use sdmmc_protocol::block::{ BlockBufferConfig, BlockPoll, BlockRequestId, BlockTransferDirection, BlockTransferMode, BlockTransferState, }; use sdmmc_protocol::{ - DataCommandPoll, + DataCommandPoll, OperationPoll, cmd::{Command, DataDirection}, - error::Error, + error::{Error, ErrorContext, Phase}, sdio::{ - BusWidth, ClockSpeed, HostEvent, HostEventKind, HostEventSource, SdioHost, SdioIrqHandle, - SdioIrqHost, SignalVoltage, + BusWidth, ClockSpeed, HostEvent, HostEventKind, HostEventSource, ReadyBusRequest, + SdioBusOp, SdioHost as ProtocolSdioHost, SdioIrqHandle, SdioIrqHost, SignalVoltage, + poll_ready_bus_op, submit_ready_bus_op, }, }; @@ -128,24 +134,121 @@ pub struct DataRequest<'a> { _buffer: PhantomData<&'a [u8]>, } +pub struct TransactionRequest<'a> { + owner: usize, + id: u64, + done: bool, + kind: TransactionRequestKind, + data: Option>, +} + +enum TransactionRequestKind { + Command { response: sdio_host2::ResponseType }, + Data { response: sdio_host2::ResponseType }, +} + +impl<'a> TransactionRequest<'a> { + fn command(owner: usize, id: u64, response: sdio_host2::ResponseType) -> Self { + Self { + owner, + id, + done: false, + kind: TransactionRequestKind::Command { response }, + data: None, + } + } + + fn data( + owner: usize, + id: u64, + request: DataRequest<'a>, + response: sdio_host2::ResponseType, + ) -> Self { + Self { + owner, + id, + done: false, + kind: TransactionRequestKind::Data { response }, + data: Some(request), + } + } +} + +pub struct BusRequest { + owner: usize, + id: u64, + done: bool, + state: BusRequestState, +} + +impl BusRequest { + fn pending(owner: usize, id: u64, state: BusRequestState) -> Self { + Self { + owner, + id, + done: false, + state, + } + } +} + +enum BusRequestState { + ResetAll(PhytiumResetState), + ResetDataLine { started: bool, polls: u32 }, + PowerOn, + PowerOff, + SetClock(PhytiumClockState), + SetBusWidth(BusWidth), + SetSignalVoltage(PhytiumVoltageState), +} + +enum PhytiumResetState { + Start, + WaitReset { polls: u32 }, + InitClock(PhytiumClockState), +} + +enum PhytiumClockState { + Start { + timing: timing::TimingTable, + }, + WaitExternalClock { + polls: u32, + timing: timing::TimingTable, + }, + WaitDisable { + polls: u32, + timing: timing::TimingTable, + }, + ProgramDivider { + timing: timing::TimingTable, + }, + WaitEnable { + polls: u32, + }, +} + +enum PhytiumVoltageState { + Start(SignalVoltage), + WaitUpdate { polls: u32 }, +} + +const PHYTIUM_RESET_POLLS: u32 = 1_000_000; +const PHYTIUM_CLOCK_POLLS: u32 = 1_000_000; + /// Cloneable, sync-safe Phytium MCI IRQ top-half handle. #[derive(Clone)] pub struct PhytiumMciIrqHandle { - pub(crate) regs: volatile::VolatilePtr<'static, crate::regs::RegisterBlock>, - pub(crate) irq_state: *const host::IrqState, + pub(crate) irq: Arc, } -// SAFETY: The handle only performs volatile MMIO accesses and atomic cache -// updates. The owning `PhytiumMci` outlives handles created by OS glue. -unsafe impl Send for PhytiumMciIrqHandle {} -// SAFETY: See the `Send` impl. -unsafe impl Sync for PhytiumMciIrqHandle {} - -impl SdioHost for PhytiumMci { +impl ProtocolSdioHost for PhytiumMci { type Event = Event; type DataRequest<'a> = DataRequest<'a>; + type BusRequest = ReadyBusRequest; fn submit_command(&mut self, cmd: &Command) -> Result<(), Error> { + self.check_not_poisoned()?; PhytiumMci::submit_command(self, cmd) } @@ -162,13 +265,13 @@ impl SdioHost for PhytiumMci { ) -> Result, Error> { let buffer = NonNull::new(buf.as_mut_ptr()).ok_or(Error::InvalidArgument)?; let mut slot = BlockRequestSlot::default(); - let request = self.submit_fifo_data_request( + let request = submit_read_with_dma_fifo_fallback( + self, cmd, buffer, buf.len(), block_size, block_count, - DataDirection::Read, &mut slot, )?; let id = request.id(); @@ -189,13 +292,13 @@ impl SdioHost for PhytiumMci { ) -> Result, Error> { let buffer = NonNull::new(buf.as_ptr() as *mut u8).ok_or(Error::InvalidArgument)?; let mut slot = BlockRequestSlot::default(); - let request = self.submit_fifo_data_request( + let request = submit_write_with_dma_fifo_fallback( + self, cmd, buffer, buf.len(), block_size, block_count, - DataDirection::Write, &mut slot, )?; let id = request.id(); @@ -240,6 +343,14 @@ impl SdioHost for PhytiumMci { fn handle_irq(&mut self) -> Self::Event { self.irq_handle().handle_irq() } + + fn submit_bus_op(&mut self, op: SdioBusOp) -> Result { + submit_ready_bus_op(self, op) + } + + fn poll_bus_op(&mut self, request: &mut Self::BusRequest) -> Result, Error> { + poll_ready_bus_op(request) + } } impl SdioIrqHost for PhytiumMci { @@ -254,15 +365,846 @@ impl SdioIrqHost for PhytiumMci { } } +impl sdio_host2::SdioHost for PhytiumMci { + type TransactionRequest<'a> + = TransactionRequest<'a> + where + Self: 'a; + type BusRequest = BusRequest; + + unsafe fn submit_transaction<'a>( + &mut self, + transaction: sdio_host2::Transaction<'a>, + ) -> Result, sdio_host2::Error> + where + Self: 'a, + { + self.check_not_poisoned().map_err(map_protocol_error)?; + if !self.physical_bus_idle() { + return Err(sdio_host2::Error::Busy); + } + let owner = self.host2_owner(); + let id = self.start_host2_request(); + let response = transaction.command.response; + match transaction.data { + None => { + if let Err(err) = self.submit_command(&transaction.command) { + self.finish_host2_request(id); + return Err(map_protocol_error(err)); + } + Ok(TransactionRequest::command(owner, id, response)) + } + Some(phase) => { + phase + .validate() + .inspect_err(|_| self.finish_host2_request(id))?; + let block_size = u32::from(phase.block_size.get()); + let block_count = phase.block_count.get(); + let request = match phase.buffer { + sdio_host2::DataBuffer::Read(buf) => { + if !matches!(phase.direction, sdio_host2::DataDirection::Read) { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + ::submit_read_data( + self, + &transaction.command, + buf, + block_size, + block_count, + ) + } + sdio_host2::DataBuffer::Write(buf) => { + if !matches!(phase.direction, sdio_host2::DataDirection::Write) { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + ::submit_write_data( + self, + &transaction.command, + buf, + block_size, + block_count, + ) + } + sdio_host2::DataBuffer::Dma(_) => { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + } + .inspect_err(|_| self.finish_host2_request(id)) + .map_err(map_protocol_error)?; + Ok(TransactionRequest::data(owner, id, request, response)) + } + } + } + + unsafe fn submit_transaction_owned<'a>( + &mut self, + transaction: sdio_host2::Transaction<'a>, + ) -> Result, sdio_host2::SubmitTransactionError<'a>> + where + Self: 'a, + { + if let Err(err) = self.check_not_poisoned() { + return Err(sdio_host2::SubmitTransactionError::new( + map_protocol_error(err), + transaction, + )); + } + if !matches!( + transaction.data.as_ref().map(|data| &data.buffer), + Some(sdio_host2::DataBuffer::Dma(_)) + ) { + return unsafe { self.submit_transaction(transaction) } + .map_err(sdio_host2::SubmitTransactionError::consumed); + } + if !self.physical_bus_idle() { + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Busy, + transaction, + )); + } + + let owner = self.host2_owner(); + let host2_id = self.start_host2_request(); + let response = transaction.command.response; + let Some(phase) = transaction.data else { + unreachable!("DMA transaction must contain a data phase") + }; + let block_size = u32::from(phase.block_size.get()); + let block_count = phase.block_count.get(); + let sdio_host2::DataBuffer::Dma(buffer) = phase.buffer else { + unreachable!("checked for DMA data buffer above") + }; + if !should_try_dma( + &transaction.command, + block_size, + block_count, + buffer.len().get(), + match phase.direction { + sdio_host2::DataDirection::Read => DataDirection::Read, + sdio_host2::DataDirection::Write => DataDirection::Write, + _ => { + self.finish_host2_request(host2_id); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + sdio_host2::Transaction::with_data(transaction.command, data), + )); + } + }, + ) { + self.finish_host2_request(host2_id); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + sdio_host2::Transaction::with_data(transaction.command, data), + )); + } + let Some(dma) = self.dma.clone() else { + self.finish_host2_request(host2_id); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + sdio_host2::Transaction::with_data(transaction.command, data), + )); + }; + let mut slot = BlockRequestSlot::default(); + let submit = match phase.direction { + sdio_host2::DataDirection::Read => self.submit_prepared_read_blocks( + transaction.command.argument, + buffer, + &dma, + &mut slot, + ), + sdio_host2::DataDirection::Write => self.submit_prepared_write_blocks( + transaction.command.argument, + buffer, + &dma, + &mut slot, + ), + _ => unreachable!("unsupported direction returned before submit"), + }; + match submit { + Ok(request) => { + let id = request.id(); + let data = DataRequest { + id, + request: Some(request), + slot, + _buffer: PhantomData, + }; + Ok(TransactionRequest::data(owner, host2_id, data, response)) + } + Err(err) => { + self.finish_host2_request(host2_id); + let error = err.error; + let buffer = err.into_buffer(); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + Err(sdio_host2::SubmitTransactionError::new( + map_protocol_error(error), + sdio_host2::Transaction::with_data(transaction.command, data), + )) + } + } + } + + fn poll_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result, sdio_host2::PollRequestError> + where + Self: 'a, + { + self.check_host2_transaction_request(request)?; + match request.kind { + TransactionRequestKind::Command { response } => { + match ::poll_command_response(self) { + Ok(sdmmc_protocol::CommandResponsePoll::Pending) => { + Ok(sdio_host2::RequestPoll::Pending) + } + Ok(sdmmc_protocol::CommandResponsePoll::Complete(resp)) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok( + resp.to_raw_response(response) + ))) + } + Ok(_) => Ok(sdio_host2::RequestPoll::Pending), + Err(err) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(map_protocol_error(err)))) + } + } + } + TransactionRequestKind::Data { response } => { + let Some(data) = request.data.as_mut() else { + let recovery = self.abort_host2_transaction_request(request).err(); + return Ok(sdio_host2::RequestPoll::Ready(Err( + recovery.unwrap_or(sdio_host2::Error::InvalidArgument) + ))); + }; + match ::poll_data_request(self, data) { + Ok(DataCommandPoll::Pending) => Ok(sdio_host2::RequestPoll::Pending), + Ok(DataCommandPoll::Complete(resp)) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok( + resp.to_raw_response(response) + ))) + } + Ok(_) => Ok(sdio_host2::RequestPoll::Pending), + Err(err) => { + let _ = self.abort_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(map_protocol_error(err)))) + } + } + } + } + } + + fn abort_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result<(), sdio_host2::Error> + where + Self: 'a, + { + if request.done { + return Ok(()); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::Error::InvalidArgument); + } + self.abort_host2_transaction_request(request) + } + + fn take_completed_dma<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Option + where + Self: 'a, + { + request + .data + .as_mut() + .and_then(|data| data.slot.take_completed_dma()) + } + + unsafe fn submit_bus_op( + &mut self, + op: sdio_host2::BusOp, + ) -> Result { + self.check_not_poisoned().map_err(map_protocol_error)?; + if !self.physical_bus_idle() { + return Err(sdio_host2::Error::Busy); + } + let state = self.prepare_host2_bus_op(op)?; + let owner = self.host2_owner(); + let id = self.start_host2_request(); + Ok(BusRequest::pending(owner, id, state)) + } + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, sdio_host2::PollRequestError> { + self.check_host2_bus_request(request)?; + match self.poll_host2_bus_state(&mut request.state) { + Ok(sdio_host2::RequestPoll::Pending) => Ok(sdio_host2::RequestPoll::Pending), + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) => { + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + Ok(sdio_host2::RequestPoll::Ready(Err(err))) => { + let _ = self.abort_host2_bus_state(&mut request.state); + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(err))) + } + Err(err) => { + let _ = self.abort_host2_bus_state(&mut request.state); + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(err))) + } + } + } + + fn abort_bus_op(&mut self, request: &mut Self::BusRequest) -> Result<(), sdio_host2::Error> { + if request.done { + return Ok(()); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::Error::InvalidArgument); + } + let result = self.abort_host2_bus_state(&mut request.state); + request.done = true; + self.finish_host2_request(request.id); + result + } +} + +impl PhytiumMci { + fn physical_bus_idle(&self) -> bool { + matches!(self.command_state, command::CommandState::Idle) + && self.pending_data.is_none() + && self.data_blocks_remaining == 0 + && self.host2_active_id.is_none() + } + + fn start_host2_request(&mut self) -> u64 { + let id = self.host2_next_id; + self.host2_next_id = self.host2_next_id.wrapping_add(1); + self.host2_active_id = Some(id); + id + } + + fn host2_owner(&self) -> usize { + self.base_addr + } + + fn finish_host2_request(&mut self, id: u64) { + if self.host2_active_id == Some(id) { + self.host2_active_id = None; + } + } + + fn prepare_host2_bus_op( + &self, + op: sdio_host2::BusOp, + ) -> Result { + match op { + sdio_host2::BusOp::ResetAll => Ok(BusRequestState::ResetAll(PhytiumResetState::Start)), + sdio_host2::BusOp::ResetCommandLine => Err(sdio_host2::Error::Unsupported), + sdio_host2::BusOp::ResetDataLine => Ok(BusRequestState::ResetDataLine { + started: false, + polls: 0, + }), + sdio_host2::BusOp::PowerOn => Ok(BusRequestState::PowerOn), + sdio_host2::BusOp::PowerOff => Ok(BusRequestState::PowerOff), + sdio_host2::BusOp::SetClock(speed) => { + let timing = + timing::TimingTable::sd_for_speed(speed).map_err(map_protocol_error)?; + Ok(BusRequestState::SetClock(PhytiumClockState::Start { + timing, + })) + } + sdio_host2::BusOp::SetClockHz(_) => Err(sdio_host2::Error::Unsupported), + sdio_host2::BusOp::SetBusWidth(width) => match width { + BusWidth::Bit1 | BusWidth::Bit4 | BusWidth::Bit8 => { + Ok(BusRequestState::SetBusWidth(width)) + } + _ => Err(sdio_host2::Error::Unsupported), + }, + sdio_host2::BusOp::SetSignalVoltage(voltage) => { + uhs_bits_after_voltage(self.regs.uhs().read(), voltage) + .map_err(map_protocol_error)?; + Ok(BusRequestState::SetSignalVoltage( + PhytiumVoltageState::Start(voltage), + )) + } + sdio_host2::BusOp::ExecuteTuning { .. } => Err(sdio_host2::Error::Unsupported), + _ => Err(sdio_host2::Error::Unsupported), + } + } + + fn poll_host2_bus_state( + &mut self, + state: &mut BusRequestState, + ) -> Result, sdio_host2::Error> { + match state { + BusRequestState::ResetAll(reset) => self.poll_host2_reset_all(reset), + BusRequestState::ResetDataLine { started, polls } => { + self.poll_host2_fifo_reset(started, polls) + } + BusRequestState::PowerOn => { + self.regs.pwren().write(1); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::PowerOff => { + self.regs.pwren().write(0); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::SetClock(clock) => self.poll_host2_clock(clock), + BusRequestState::SetBusWidth(width) => { + PhytiumMci::set_bus_width(self, *width); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::SetSignalVoltage(voltage) => self.poll_host2_voltage(voltage), + } + } + + fn poll_host2_reset_all( + &mut self, + state: &mut PhytiumResetState, + ) -> Result, sdio_host2::Error> { + match state { + PhytiumResetState::Start => { + self.regs.clkena().write(crate::regs::ClkEna::new()); + self.regs.ctrl().update(|r| { + r.with_use_internal_dmac(false) + .with_dma_enable(false) + .with_int_enable(false) + }); + self.regs.ctrl().update(|r| { + r.with_controller_reset(true) + .with_fifo_reset(true) + .with_dma_reset(true) + }); + *state = PhytiumResetState::WaitReset { polls: 0 }; + Ok(sdio_host2::RequestPoll::Pending) + } + PhytiumResetState::WaitReset { polls } => { + let ctrl = self.regs.ctrl().read(); + if !ctrl.controller_reset() && !ctrl.fifo_reset() && !ctrl.dma_reset() { + self.regs.intmask().write(0); + self.regs.idinten().write(0); + self.clear_all_int_status(); + self.regs.idsts().write(u32::MAX); + self.irq.state.clear_all(); + self.clear_completion_irq_enabled(); + self.regs.ctype().write(crate::regs::CType::new()); + self.regs.uhs().write(crate::regs::Uhs::new()); + self.regs.tmout().write(0xffff_ffff); + self.regs.pwren().write(1); + self.regs.fifoth().write(crate::host::FIFO_THRESHOLD); + self.write_ext_reg( + crate::regs::CARD_THRCTL_OFFSET, + crate::host::CARD_READ_THRESHOLD_ENABLE + | crate::host::CARD_READ_THRESHOLD_DEPTH8, + ); + *state = PhytiumResetState::InitClock(PhytiumClockState::Start { + timing: timing::TimingTable::sd_for_speed(ClockSpeed::Identification) + .map_err(map_protocol_error)?, + }); + return Ok(sdio_host2::RequestPoll::Pending); + } + if *polls >= PHYTIUM_RESET_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new( + Phase::Init, + )))); + } + *polls += 1; + Ok(sdio_host2::RequestPoll::Pending) + } + PhytiumResetState::InitClock(clock) => self.poll_host2_clock(clock), + } + } + + fn poll_host2_fifo_reset( + &mut self, + started: &mut bool, + polls: &mut u32, + ) -> Result, sdio_host2::Error> { + if !*started { + self.regs.ctrl().update(|r| r.with_fifo_reset(true)); + *started = true; + } + if !self.regs.ctrl().read().fifo_reset() { + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + if *polls >= PHYTIUM_RESET_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new( + Phase::DataRead, + )))); + } + *polls += 1; + Ok(sdio_host2::RequestPoll::Pending) + } + + fn poll_host2_clock( + &mut self, + state: &mut PhytiumClockState, + ) -> Result, sdio_host2::Error> { + match state { + PhytiumClockState::Start { timing } => { + self.use_hold_reg = timing.use_hold; + self.write_ext_reg(crate::regs::CLK_SRC_OFFSET, 0); + self.write_ext_reg(crate::regs::CLK_SRC_OFFSET, timing.clk_src); + *state = PhytiumClockState::WaitExternalClock { + polls: 0, + timing: *timing, + }; + Ok(sdio_host2::RequestPoll::Pending) + } + PhytiumClockState::WaitExternalClock { polls, timing } => { + if self.regs.cksts().read().ready() { + self.regs.clkena().write(crate::regs::ClkEna::new()); + self.start_update_clock(false); + *state = PhytiumClockState::WaitDisable { + polls: 0, + timing: *timing, + }; + return Ok(sdio_host2::RequestPoll::Pending); + } + if *polls >= PHYTIUM_CLOCK_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new( + Phase::Init, + )))); + } + *polls += 1; + Ok(sdio_host2::RequestPoll::Pending) + } + PhytiumClockState::WaitDisable { polls, timing } => { + if self.poll_update_clock_complete(polls)? { + *state = PhytiumClockState::ProgramDivider { timing: *timing }; + } + Ok(sdio_host2::RequestPoll::Pending) + } + PhytiumClockState::ProgramDivider { timing } => { + self.regs.clkdiv().write(timing.clk_div); + self.regs + .clkena() + .write(crate::regs::ClkEna::new().with_cclk_enable(1)); + self.start_update_clock(false); + *state = PhytiumClockState::WaitEnable { polls: 0 }; + Ok(sdio_host2::RequestPoll::Pending) + } + PhytiumClockState::WaitEnable { polls } => { + if self.poll_update_clock_complete(polls)? { + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + Ok(sdio_host2::RequestPoll::Pending) + } + } + } + + fn poll_host2_voltage( + &mut self, + state: &mut PhytiumVoltageState, + ) -> Result, sdio_host2::Error> { + match state { + PhytiumVoltageState::Start(voltage) => { + let next = uhs_bits_after_voltage(self.regs.uhs().read(), *voltage) + .map_err(map_protocol_error)?; + self.regs.uhs().write(next); + self.start_update_clock(matches!(*voltage, SignalVoltage::V180)); + *state = PhytiumVoltageState::WaitUpdate { polls: 0 }; + Ok(sdio_host2::RequestPoll::Pending) + } + PhytiumVoltageState::WaitUpdate { polls } => { + if self.poll_update_clock_complete(polls)? { + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + Ok(sdio_host2::RequestPoll::Pending) + } + } + } + + fn start_update_clock(&self, voltage_switch: bool) { + self.regs.cmd().write( + crate::regs::Cmd::new() + .with_start_cmd(true) + .with_wait_prvdata_complete(true) + .with_update_clock_registers_only(true) + .with_volt_switch(voltage_switch), + ); + } + + fn poll_update_clock_complete(&self, polls: &mut u32) -> Result { + if !self.regs.cmd().read().start_cmd() { + return Ok(true); + } + if *polls >= PHYTIUM_CLOCK_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new( + Phase::Init, + )))); + } + *polls += 1; + Ok(false) + } + + fn check_host2_transaction_request( + &self, + request: &TransactionRequest<'_>, + ) -> Result<(), sdio_host2::PollRequestError> { + if request.done { + return Err(sdio_host2::PollRequestError::AlreadyCompleted); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::PollRequestError::WrongOwner); + } + if self.host2_active_id != Some(request.id) { + return Err(sdio_host2::PollRequestError::StaleGeneration); + } + Ok(()) + } + + fn check_host2_bus_request( + &self, + request: &BusRequest, + ) -> Result<(), sdio_host2::PollRequestError> { + if request.done { + return Err(sdio_host2::PollRequestError::AlreadyCompleted); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::PollRequestError::WrongOwner); + } + if self.host2_active_id != Some(request.id) { + return Err(sdio_host2::PollRequestError::StaleGeneration); + } + Ok(()) + } + + fn complete_host2_transaction_request(&mut self, request: &mut TransactionRequest<'_>) { + request.done = true; + self.finish_host2_request(request.id); + } + + fn complete_host2_bus_request(&mut self, request: &mut BusRequest) { + request.done = true; + self.finish_host2_request(request.id); + } + + fn abort_host2_bus_state( + &mut self, + state: &mut BusRequestState, + ) -> Result<(), sdio_host2::Error> { + match state { + BusRequestState::ResetAll(_) + | BusRequestState::SetClock(_) + | BusRequestState::SetSignalVoltage(_) => { + self.reset_and_init_preserving_irq() + .map_err(map_protocol_error)?; + } + BusRequestState::ResetDataLine { started, .. } if *started => { + self.reset_fifo(sdmmc_protocol::Phase::DataRead) + .map_err(map_protocol_error)?; + } + BusRequestState::PowerOn + | BusRequestState::PowerOff + | BusRequestState::SetBusWidth(_) => {} + BusRequestState::ResetDataLine { .. } => {} + } + self.pending_data = None; + self.data_blocks_remaining = 0; + self.command_state = command::CommandState::Idle; + Ok(()) + } + + fn abort_host2_transaction_request( + &mut self, + request: &mut TransactionRequest<'_>, + ) -> Result<(), sdio_host2::Error> { + let result = if let Some(data) = request.data.as_mut() { + if let Some(active) = data.request.take() { + let id = active.id(); + let mut pending = Some(active); + self.abort_block_request_response(&mut pending, id, &mut data.slot) + .map_err(map_protocol_error) + } else { + Ok(()) + } + } else { + self.abort_command().map_err(map_protocol_error) + }; + request.done = true; + self.finish_host2_request(request.id); + result + } +} + +fn map_protocol_error(err: Error) -> sdio_host2::Error { + match err { + Error::Timeout(_) => sdio_host2::Error::Timeout, + Error::Crc(_) => sdio_host2::Error::Crc, + Error::NoCard => sdio_host2::Error::NoCard, + Error::Busy => sdio_host2::Error::Busy, + Error::UnsupportedCommand => sdio_host2::Error::Unsupported, + Error::Misaligned => sdio_host2::Error::Misaligned, + Error::InvalidArgument => sdio_host2::Error::InvalidArgument, + Error::BusError(_) => sdio_host2::Error::Bus, + Error::ReadError(_) | Error::WriteError(_) | Error::BadResponse(_) => { + sdio_host2::Error::Bus + } + Error::CardError(_) | Error::CardLocked => sdio_host2::Error::Controller, + _ => sdio_host2::Error::Controller, + } +} + +fn submit_read_with_dma_fifo_fallback( + host: &mut PhytiumMci, + cmd: &Command, + buffer: NonNull, + len: usize, + block_size: u32, + block_count: u32, + slot: &mut BlockRequestSlot, +) -> Result { + if should_try_dma(cmd, block_size, block_count, len, DataDirection::Read) + && let Some(dma) = host.dma.clone() + { + match host.submit_read_blocks( + cmd.argument, + buffer, + NonZeroUsize::new(len).ok_or(Error::InvalidArgument)?, + Some(&dma), + BlockTransferMode::Dma, + slot, + ) { + Ok(request) => return Ok(request), + Err(err) if can_fallback_to_fifo(err) => {} + Err(err) => return Err(err), + } + } + + host.submit_fifo_data_request( + cmd, + buffer, + len, + block_size, + block_count, + DataDirection::Read, + slot, + ) +} + +fn submit_write_with_dma_fifo_fallback( + host: &mut PhytiumMci, + cmd: &Command, + buffer: NonNull, + len: usize, + block_size: u32, + block_count: u32, + slot: &mut BlockRequestSlot, +) -> Result { + if should_try_dma(cmd, block_size, block_count, len, DataDirection::Write) + && let Some(dma) = host.dma.clone() + { + match host.submit_write_blocks( + cmd.argument, + buffer, + NonZeroUsize::new(len).ok_or(Error::InvalidArgument)?, + Some(&dma), + BlockTransferMode::Dma, + slot, + ) { + Ok(request) => return Ok(request), + Err(err) if can_fallback_to_fifo(err) => {} + Err(err) => return Err(err), + } + } + + host.submit_fifo_data_request( + cmd, + buffer, + len, + block_size, + block_count, + DataDirection::Write, + slot, + ) +} + +fn should_try_dma( + cmd: &Command, + block_size: u32, + block_count: u32, + len: usize, + direction: DataDirection, +) -> bool { + block_size == 512 + && len == block_count as usize * 512 + && matches!( + (direction, cmd.index), + (DataDirection::Read, 17 | 18) | (DataDirection::Write, 24 | 25) + ) +} + +fn can_fallback_to_fifo(err: Error) -> bool { + matches!( + err, + Error::UnsupportedCommand | Error::InvalidArgument | Error::Misaligned + ) +} + +impl PhytiumMci { + pub fn block_buffer_config(&self, mode: BlockTransferMode) -> BlockBufferConfig { + match mode { + BlockTransferMode::Fifo => { + BlockBufferConfig::new(NonZeroUsize::new(512).unwrap(), 1, None) + } + BlockTransferMode::Dma => { + BlockBufferConfig::new(NonZeroUsize::new(512).unwrap(), 512, Some(self.dma_mask)) + } + // Future BlockTransferMode variants fall back to the conservative Fifo config. + _ => BlockBufferConfig::new(NonZeroUsize::new(512).unwrap(), 1, None), + } + } +} + #[cfg(test)] mod tests { + use core::num::{NonZeroU16, NonZeroU32}; + use sdmmc_protocol::{ + BlockTransferMode, cmd::CMD0, response::ResponseType, sdio::{ClockSpeed, SignalVoltage}, }; use crate::{ + PhytiumMci, command::encode_command, regs::{Ctrl, Uhs}, timing::{MediaKind, TimingTable}, @@ -308,11 +1250,7 @@ mod tests { #[test] fn r3_command_encoding_does_not_enable_crc_check() { - let cmd = sdmmc_protocol::cmd::Command { - cmd: 1, - arg: 0, - resp_type: ResponseType::R3, - }; + let cmd = sdmmc_protocol::cmd::Command::new(1, 0, ResponseType::R3); let reg = encode_command(&cmd, None); assert!(reg.response_expect()); assert!(!reg.check_response_crc()); @@ -342,13 +1280,55 @@ mod tests { #[test] fn command_register_keeps_hold_register_optional() { - let cmd = sdmmc_protocol::cmd::Command { - cmd: 17, - arg: 0, - resp_type: ResponseType::R1, - }; + let cmd = sdmmc_protocol::cmd::Command::new(17, 0, ResponseType::R1); let without_hold = encode_command(&cmd, None).with_use_hold_reg(false); assert!(!without_hold.use_hold_reg()); assert_eq!(without_hold.cmd_index(), 17); } + + #[test] + fn host2_data_submit_reports_busy_without_dirtying_pending_data() { + let mut host = unsafe { PhytiumMci::new_from_addr(0x1000_0000) }; + host.command_state = crate::command::CommandState::Issued { + cmd: sdmmc_protocol::cmd::Command::new(0, 0, ResponseType::None), + polls: 0, + }; + let mut buf = [0u8; 512]; + let data = sdio_host2::DataPhase::read( + NonZeroU16::new(512).unwrap(), + NonZeroU32::new(1).unwrap(), + &mut buf, + ) + .unwrap(); + let tx = sdio_host2::Transaction::with_data( + sdmmc_protocol::cmd::Command::new(17, 0, ResponseType::R1), + data, + ); + + let err = match unsafe { + ::submit_transaction(&mut host, tx) + } { + Ok(_) => panic!("busy host accepted a second transaction"), + Err(err) => err, + }; + + assert_eq!(err, sdio_host2::Error::Busy); + assert!(host.pending_data.is_none()); + assert_eq!(host.data_blocks_remaining, 0); + } + + #[test] + fn exposes_block_buffer_constraints() { + let host = unsafe { PhytiumMci::new_from_addr(0x1000_0000) }; + + let fifo = host.block_buffer_config(BlockTransferMode::Fifo); + assert_eq!(fifo.block_size.get(), 512); + assert_eq!(fifo.align, 1); + assert_eq!(fifo.dma_mask, None); + + let dma = host.block_buffer_config(BlockTransferMode::Dma); + assert_eq!(dma.block_size.get(), 512); + assert_eq!(dma.align, 512); + assert_eq!(dma.dma_mask, Some(u32::MAX as u64)); + } } diff --git a/drivers/blk/phytium-mci-host/src/rdif.rs b/drivers/blk/phytium-mci-host/src/rdif.rs new file mode 100644 index 0000000000..7724e0aa3c --- /dev/null +++ b/drivers/blk/phytium-mci-host/src/rdif.rs @@ -0,0 +1,56 @@ +//! RDIF block-device adapter for [`PhytiumMci`]. + +use dma_api::DeviceDma; +pub use protocol_rdif::{BlockConfig, BlockDevice, BlockQueue}; +pub use rdif_block::{ + BInterface, BIrqHandler, BOwnedQueue, BQueue, BlkError, IQueue, IQueueOwned, Interface, + IrqHandlerHandle, IrqHandlerSlot, OwnedRequest, PollError, QueueHandle, Request, + RequestId as RdifRequestId, RequestPoll as OwnedRequestPoll, RequestStatus, SubmitError, +}; +use sdmmc_protocol::{ + rdif as protocol_rdif, + sdio::{SdioHost2Adapter, SdioSdmmc}, +}; + +use crate::PhytiumMci; + +pub fn device( + card: SdioSdmmc>, + config: BlockConfig, +) -> BlockDevice> { + BlockDevice::new(card, config) +} + +pub fn dma_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, + dma: DeviceDma, +) -> BlockConfig { + BlockConfig::dma(name, capacity_blocks, irq_driven, dma) + .with_max_blocks_per_request(1024) + .with_max_segment_size(1024 * protocol_rdif::BLOCK_SIZE) +} + +pub const fn fifo_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, +) -> BlockConfig { + BlockConfig::fifo(name, capacity_blocks, irq_driven) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fifo_config_keeps_one_block_limits() { + let config = fifo_config("phytium-mci", 16, true); + let limits = protocol_rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, 1); + assert_eq!(limits.max_segment_size, protocol_rdif::BLOCK_SIZE); + assert!(!config.uses_dma()); + } +} diff --git a/drivers/blk/sdhci-host/Cargo.toml b/drivers/blk/sdhci-host/Cargo.toml index 638a10d006..7e0452a666 100644 --- a/drivers/blk/sdhci-host/Cargo.toml +++ b/drivers/blk/sdhci-host/Cargo.toml @@ -10,11 +10,13 @@ keywords = ["sd", "sdhci", "embedded", "no_std"] categories = ["embedded", "no-std", "hardware-support"] [dependencies] -sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio"] } +sdmmc-protocol = { workspace = true, default-features = false, features = ["sdio", "rdif"] } +sdio-host2.workspace = true embedded-hal = "1" log.workspace = true dma-api.workspace = true mmio-api.workspace = true +rdif-block.workspace = true [features] default = [] diff --git a/drivers/blk/sdhci-host/README.md b/drivers/blk/sdhci-host/README.md index b6462298c9..5e7e939c1e 100644 --- a/drivers/blk/sdhci-host/README.md +++ b/drivers/blk/sdhci-host/README.md @@ -3,15 +3,15 @@ `no_std` SD Host Controller Interface (SDHCI v3.x) backend for [`sdmmc-protocol`](../sdmmc-protocol). -This crate plugs SDHCI register programming into the -`sdmmc_protocol::sdio::SdioHost` trait so `SdioSdmmc` can drive a real -controller. Platform code is still responsible for MMIO mapping, clock/reset +This crate plugs SDHCI register programming into the physical +`sdio_host2::SdioHost` trait so `SdioSdmmc::new_host2` can drive a real +controller through `sdmmc-protocol`. Platform code is still responsible for MMIO mapping, clock/reset tree setup, power rails, pinmux, IRQ routing, and DMA cache coherency. ## Status - Compiles as a `no_std` controller backend. -- Intended for use through `sdmmc_protocol::sdio::SdioSdmmc`. +- Intended for use through `sdmmc_protocol::sdio::SdioSdmmc::new_host2`. - Board-specific clock, power, pinmux, and DMA policy must be supplied by the caller. - Real hardware bring-up still depends on the surrounding SoC integration. @@ -45,12 +45,10 @@ use sdhci_host::Sdhci; // caller has exclusive access to. let mmio = NonNull::new(0xFE31_0000 as *mut u8).unwrap(); let mut host = unsafe { Sdhci::new(mmio) }; -host.reset_all()?; -host.set_power(0x0f); -host.enable_interrupts(); -host.enable_clock(150_000_000, 400_000)?; +// Optional platform capabilities such as HostClock, HostResetHook, DMA, and +// 1.8 V support are installed here before the protocol layer owns the host. -let mut card = SdioSdmmc::new(host); +let mut card = SdioSdmmc::new_host2(host); let mut scratch = SdioInitScratch::new(); let mut request = card.submit_init(&mut scratch)?; while let OperationPoll::Pending = card.poll_init_request(&mut request)? { @@ -66,7 +64,10 @@ the driver. ## Block Request Usage -Use `Sdhci::submit_read_blocks` / `Sdhci::submit_write_blocks`, then drive +Normal block-device integration should use `sdhci_host::rdif::device`, which +routes RDIF requests through `sdmmc-protocol` and the native `sdio-host2` +transaction path. The lower-level primitives remain available for controller +bring-up: use `Sdhci::submit_read_blocks` / `Sdhci::submit_write_blocks`, then drive completion with `Sdhci::poll_block_request`. `BlockTransferMode::Dma` uses ADMA2 and owns request-buffer mapping, descriptor allocation, descriptor cache sync, and completion sync. `BlockTransferMode::Fifo` uses the FIFO @@ -79,7 +80,7 @@ use dma_api::DeviceDma; use sdhci_host::{BlockRequestSlot, BlockTransferMode, RequestId, Sdhci}; # use platform::DmaImpl; -let dma = DeviceDma::new(u32::MAX as u64, &DmaImpl); +let dma = DeviceDma::new_legacy(u32::MAX as u64, &DmaImpl); let mut host = unsafe { Sdhci::new_from_addr(0xFE31_0000) }; let mut block = [0u8; 512]; let ptr = NonNull::new(block.as_mut_ptr()).unwrap(); @@ -107,18 +108,22 @@ block-buffer contract. 2. Configure the platform clock so the controller has a viable reference clock before calling `Sdhci::new` (RK3568 needs the CRU bringing `CLK_EMMC_CORE` up at ≥ 25 MHz). -3. `host.reset_all()?` — clears CMD/DAT inhibits and the interrupt - registers. -4. `host.set_power(POWER_330)` (or whatever your card needs). -5. `host.enable_interrupts()` — enables status flags. The driver polls; - it does NOT enable signal-level IRQ delivery. -6. `host.enable_clock(base_hz, 400_000)` — start at 400 kHz for - identification. -7. Build `SdioSdmmc::new(host)`, submit initialization with +3. Install optional capabilities such as `Sdhci::set_external_clock`, + `Sdhci::set_reset_hook`, `Sdhci::set_dma`, and + `Sdhci::enable_1v8_signaling` before handing the host to the protocol + layer. +4. Build `SdioSdmmc::new_host2(host)`, submit initialization with `submit_init`, and drive it with `poll_init_request`. The protocol - layer will ramp the clock up to 25 MHz / 50 MHz via `set_clock`; - platform/runtime code chooses whether pending work spins, yields, or - waits for an IRQ. + layer starts with native `sdio-host2` bus operations for `ResetAll`, + `PowerOn`, initial voltage, 1-bit bus width, and 400 kHz identification + clock before issuing SD/MMC commands, then ramps the card to 25 MHz / + 50 MHz via later bus ops. Platform/runtime code chooses whether pending + work spins, yields, or waits for an IRQ. + +The lower-level blocking helpers such as `Sdhci::reset_all`, +`Sdhci::set_power`, and `Sdhci::enable_clock` remain useful for diagnostics, +but normal card initialization should let `SdioSdmmc::new_host2` drive those +steps through submit/poll bus operations. If the SoC requires external clock-tree programming for each SD speed, implement `sdhci_host::HostClock` in platform glue and register it with diff --git a/drivers/blk/sdhci-host/src/command.rs b/drivers/blk/sdhci-host/src/command.rs index 7c5759c710..b05b8a9431 100644 --- a/drivers/blk/sdhci-host/src/command.rs +++ b/drivers/blk/sdhci-host/src/command.rs @@ -23,10 +23,17 @@ pub(crate) enum CommandState { cmd: Command, data: Option, use_dma: bool, + polls: u32, }, Issued { cmd: Command, data_line: bool, + polls: u32, + }, + WaitingBusy { + cmd: Command, + response: Response, + polls: u32, }, Complete { response: Response, @@ -61,11 +68,13 @@ impl Sdhci { } let data = self.pending_data.take(); info_command_start(self, cmd, data); + self.prepare_irq_for_request(); self.command_state = CommandState::WaitingInhibit { cmd: *cmd, data, use_dma: self.use_dma, + polls: 0, }; if let Err(err) = self.poll_command() { self.command_state = CommandState::Idle; @@ -77,26 +86,55 @@ impl Sdhci { /// Advance the currently submitted command without blocking. pub fn poll_command(&mut self) -> Result { match self.command_state { - CommandState::WaitingInhibit { cmd, data, use_dma } => { + CommandState::WaitingInhibit { + cmd, + data, + use_dma, + polls, + } => { if !self.command_can_issue(&cmd, data.is_some()) { + if polls >= COMMAND_WAIT_POLLS { + let err = + Error::Timeout(ErrorContext::for_cmd(Phase::CommandSend, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::WaitingInhibit { + cmd, + data, + use_dma, + polls: polls + 1, + }; return Ok(CommandPoll::Pending); } self.program_command(&cmd, data, use_dma)?; return Ok(CommandPoll::Pending); } CommandState::Issued { .. } => {} + CommandState::WaitingBusy { + cmd, + response, + polls, + } => { + return self.poll_r1b_busy(cmd, response, polls); + } CommandState::Complete { .. } => return Ok(CommandPoll::Complete), CommandState::Failed { error, .. } => return Err(error), CommandState::Idle => return Err(Error::InvalidArgument), } - let CommandState::Issued { cmd, data_line } = self.command_state else { + let CommandState::Issued { + cmd, + data_line, + polls, + } = self.command_state + else { unreachable!(); }; let (normal, error) = self.take_command_irq_status(); if normal & NORMAL_INT_CMD_COMPLETE != 0 { - let response = match decode_response(self, cmd.resp_type) { + let response = match decode_response(self, cmd.response) { Ok(r) => r, Err(err) => { // Park the FSM in Failed before propagating: bare `?` would @@ -107,26 +145,95 @@ impl Sdhci { return Err(err); } }; - log::debug!("sdhci: CMD{} response {:?}", cmd.cmd, response); + log::debug!("sdhci: CMD{} response {:?}", cmd.index, response); + if matches!(cmd.response, ResponseType::R1b) { + self.command_state = CommandState::WaitingBusy { + cmd, + response, + polls: 0, + }; + return Ok(CommandPoll::Pending); + } self.command_state = CommandState::Complete { response }; Ok(CommandPoll::Complete) } else if normal & NORMAL_INT_ERROR != 0 { - self.log_status("command wait failed", cmd.cmd); + self.log_status("command wait failed", cmd.index); self.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CLEAR_ALL); self.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_CLEAR_ALL); let _ = self.reset_cmd(); if data_line { let _ = self.reset_dat(); } - let err = self.translate_error_bits(error & ERROR_INT_CMD_LINE_MASK, cmd.cmd); + let err = self.translate_error_bits(error & ERROR_INT_CMD_LINE_MASK, cmd.index); self.command_state = CommandState::Failed { error: err }; Err(err) } else { + if polls >= COMMAND_WAIT_POLLS { + self.log_status("command response timeout", cmd.index); + let _ = self.reset_cmd(); + if data_line { + let _ = self.reset_dat(); + } + let err = Error::Timeout(ErrorContext::for_cmd(Phase::ResponseWait, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::Issued { + cmd, + data_line, + polls: polls + 1, + }; Ok(CommandPoll::Pending) } } + fn poll_r1b_busy( + &mut self, + cmd: Command, + response: Response, + polls: u32, + ) -> Result { + let (normal, error) = self.take_command_irq_status(); + if normal & NORMAL_INT_ERROR != 0 { + self.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CLEAR_ALL); + self.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_CLEAR_ALL); + let _ = self.reset_cmd(); + let _ = self.reset_dat(); + let err = self.translate_error_bits(error & ERROR_INT_DATA_LINE_MASK, cmd.index); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + let present = self.read_u32(REG_PRESENT_STATE); + if present & PRESENT_DAT0_LINE_SIGNAL_LEVEL != 0 { + self.command_state = CommandState::Complete { response }; + return Ok(CommandPoll::Complete); + } + if polls >= COMMAND_BUSY_POLLS { + let _ = self.reset_dat(); + let err = Error::Timeout(ErrorContext::for_cmd(Phase::BusyWait, cmd.index)); + self.command_state = CommandState::Failed { error: err }; + return Err(err); + } + self.command_state = CommandState::WaitingBusy { + cmd, + response, + polls: polls + 1, + }; + Ok(CommandPoll::Pending) + } + fn take_command_irq_status(&mut self) -> (u16, u16) { + if self.completion_irq_enabled() { + let normal = self + .irq + .state + .take_normal(NORMAL_INT_CMD_COMPLETE | NORMAL_INT_ERROR); + let error = self.irq.state.take_error_all(); + if error != 0 { + self.irq.state.clear_normal(NORMAL_INT_ERROR); + } + return (normal, error); + } let normal_hw = self.read_u16(REG_NORMAL_INT_STATUS); let error_hw = if normal_hw & NORMAL_INT_ERROR != 0 { self.read_u16(REG_ERROR_INT_STATUS) @@ -141,37 +248,59 @@ impl Sdhci { self.write_u16(REG_ERROR_INT_STATUS, error_hw); } - let normal = self - .irq_state - .take_normal(NORMAL_INT_CMD_COMPLETE | NORMAL_INT_ERROR) - | normal_hw; - let error = self.irq_state.take_error_all() | error_hw; - if error != 0 { - self.irq_state.clear_normal(NORMAL_INT_ERROR); - } - (normal, error) + (normal_hw, error_hw) } pub fn take_command_response(&mut self) -> Result { match self.command_state { CommandState::Complete { response, .. } => { self.command_state = CommandState::Idle; + if self.active_data_cmd == 0 { + self.clear_cached_irq_status(); + } Ok(response) } CommandState::Failed { error, .. } => { self.command_state = CommandState::Idle; + self.clear_cached_irq_status(); Err(error) } - CommandState::Idle | CommandState::Issued { .. } => Err(Error::InvalidArgument), + CommandState::Idle | CommandState::Issued { .. } | CommandState::WaitingBusy { .. } => { + Err(Error::InvalidArgument) + } CommandState::WaitingInhibit { .. } => Err(Error::InvalidArgument), } } pub(crate) fn clear_cached_irq_status(&mut self) { - self.irq_state.clear_all(); + self.irq.state.end_request(); + } + + pub(crate) fn abort_command(&mut self) -> Result<(), Error> { + self.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CLEAR_ALL); + self.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_CLEAR_ALL); + self.clear_cached_irq_status(); + self.reset_cmd()?; + self.reset_dat()?; + self.pending_data = None; + self.use_dma = false; + self.active_data_cmd = 0; + self.command_state = CommandState::Idle; + Ok(()) } pub(crate) fn take_data_irq_status(&mut self) -> (u16, u16) { + if self.completion_irq_enabled() { + let normal = self + .irq + .state + .take_normal(NORMAL_INT_XFER_COMPLETE | NORMAL_INT_ERROR); + let error = self.irq.state.take_error_all(); + if error != 0 { + self.irq.state.clear_normal(NORMAL_INT_ERROR); + } + return (normal, error); + } let normal_hw = self.read_u16(REG_NORMAL_INT_STATUS); let error_hw = if normal_hw & NORMAL_INT_ERROR != 0 { self.read_u16(REG_ERROR_INT_STATUS) @@ -186,18 +315,18 @@ impl Sdhci { self.write_u16(REG_ERROR_INT_STATUS, error_hw); } - let normal = self - .irq_state - .take_normal(NORMAL_INT_XFER_COMPLETE | NORMAL_INT_ERROR) - | normal_hw; - let error = self.irq_state.take_error_all() | error_hw; - if error != 0 { - self.irq_state.clear_normal(NORMAL_INT_ERROR); - } - (normal, error) + (normal_hw, error_hw) } pub(crate) fn take_fifo_irq_status(&mut self, mask: u16) -> (u16, u16) { + if self.completion_irq_enabled() { + let normal = self.irq.state.take_normal(mask); + let error = self.irq.state.take_error_all(); + if mask & NORMAL_INT_ERROR != 0 && error != 0 && normal & NORMAL_INT_ERROR != 0 { + self.irq.state.clear_normal(NORMAL_INT_ERROR); + } + return (normal, error); + } let normal_hw = self.read_u16(REG_NORMAL_INT_STATUS); let consume_error = mask & NORMAL_INT_ERROR != 0; let error_hw = if consume_error && normal_hw & NORMAL_INT_ERROR != 0 { @@ -213,12 +342,13 @@ impl Sdhci { self.write_u16(REG_ERROR_INT_STATUS, error_hw); } - let normal = self.irq_state.take_normal(mask) | normal_hw; - let error = self.irq_state.take_error_all() | error_hw; - if consume_error && error != 0 && normal & NORMAL_INT_ERROR != 0 { - self.irq_state.clear_normal(NORMAL_INT_ERROR); - } - (normal, error) + (normal_hw, error_hw) + } + + fn prepare_irq_for_request(&mut self) { + self.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CLEAR_ALL); + self.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_CLEAR_ALL); + self.irq.state.begin_request(); } fn translate_error_bits(&self, err: u16, cmd_index: u8) -> Error { @@ -309,7 +439,9 @@ impl Sdhci { self.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CLEAR_ALL); self.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_CLEAR_ALL); - self.clear_cached_irq_status(); + // Keep the active request generation alive: the IRQ top-half must be + // able to cache this command/data completion for task-side poll. + self.irq.state.clear_all(); if let Some(d) = data { self.configure_data_phase(d.direction, d.block_size, d.block_count, use_dma); @@ -317,21 +449,25 @@ impl Sdhci { self.write_u16(REG_TRANSFER_MODE, 0); } - self.write_u32(REG_ARGUMENT, cmd.arg); + self.write_u32(REG_ARGUMENT, cmd.argument); let cmd_reg = encode_command(cmd, has_data)?; self.write_u16(REG_COMMAND, cmd_reg); if has_data { - self.active_data_cmd = cmd.cmd; + self.active_data_cmd = cmd.index; } - self.log_status("issued", cmd.cmd); + self.log_status("issued", cmd.index); self.command_state = CommandState::Issued { cmd: *cmd, data_line, + polls: 0, }; Ok(()) } } +const COMMAND_WAIT_POLLS: u32 = 1_000_000; +const COMMAND_BUSY_POLLS: u32 = 1_000_000; + fn transfer_mode(direction: DataDirection, block_count: u32, use_dma: bool) -> u16 { let mut mode = XFER_MODE_BLOCK_COUNT_ENABLE; if block_count > 1 { @@ -351,14 +487,14 @@ fn command_inhibit_mask(cmd: &Command, has_data: bool) -> u32 { if command_uses_data_line(cmd, has_data) { mask |= PRESENT_DAT_INHIBIT; } - if cmd.cmd == sdmmc_protocol::cmd::CMD12.cmd { + if cmd.index == sdmmc_protocol::cmd::CMD12.index { mask &= !PRESENT_DAT_INHIBIT; } mask } fn command_uses_data_line(cmd: &Command, has_data: bool) -> bool { - has_data || matches!(cmd.resp_type, ResponseType::R1b) + has_data || matches!(cmd.response, ResponseType::R1b) } fn info_command_start(host: &Sdhci, cmd: &Command, data: Option) { @@ -366,9 +502,9 @@ fn info_command_start(host: &Sdhci, cmd: &Command, data: Option log::debug!( "sdhci: CMD{} arg={:#010x} resp={:?} data={:?} blocks={} block_size={} \ present={:#010x}", - cmd.cmd, - cmd.arg, - cmd.resp_type, + cmd.index, + cmd.argument, + cmd.response, data.direction, data.block_count, data.block_size, @@ -376,16 +512,16 @@ fn info_command_start(host: &Sdhci, cmd: &Command, data: Option log::debug!( "sdhci: CMD{} arg={:#010x} resp={:?} data=none present={:#010x}", - cmd.cmd, - cmd.arg, - cmd.resp_type, + cmd.index, + cmd.argument, + cmd.response, host.read_u32(REG_PRESENT_STATE) ), } } fn encode_command(cmd: &Command, has_data: bool) -> Result { - let resp_bits: u16 = match cmd.resp_type { + let resp_bits: u16 = match cmd.response { ResponseType::None => CMD_RESP_NONE, ResponseType::R1 | ResponseType::R5 | ResponseType::R6 | ResponseType::R7 => { CMD_RESP_LEN48 | CMD_CRC_CHECK | CMD_INDEX_CHECK @@ -398,23 +534,27 @@ fn encode_command(cmd: &Command, has_data: bool) -> Result { }; let data_bit = if has_data { CMD_DATA_PRESENT } else { 0 }; - let cmd_index = (cmd.cmd as u16) << 8; + let cmd_index = (cmd.index as u16) << 8; Ok(cmd_index | data_bit | resp_bits) } fn decode_response(host: &Sdhci, resp_type: ResponseType) -> Result { Ok(match resp_type { ResponseType::None => Response::Empty, - ResponseType::R1 | ResponseType::R1b => Response::R1(R1Response { + ResponseType::R1 => Response::R1(R1Response { + raw: host.response32(0), + }), + ResponseType::R1b => Response::R1b(R1Response { raw: host.response32(0), }), ResponseType::R2 => Response::R2(read_r2(host)), ResponseType::R3 => Response::R3(OcrResponse::from_raw(host.response32(0))), - ResponseType::R4 | ResponseType::R5 => { - // SDIO IO commands aren't part of the MVP; surface them as - // "bad response" rather than silently returning zeros. - return Err(Error::BadResponse(ErrorContext::default())); - } + ResponseType::R4 => Response::R4(sdmmc_protocol::response::SdioOcrResponse::from_raw( + host.response32(0), + )), + ResponseType::R5 => Response::R5(sdmmc_protocol::response::SdioRwResponse::from_raw( + host.response32(0), + )), ResponseType::R6 => Response::R6(RcaResponse::from_raw(host.response32(0))), ResponseType::R7 => Response::R7(IfCondResponse::from_raw(host.response32(0))), // Future ResponseType variants are not decoded by this controller. @@ -455,7 +595,7 @@ fn read_r2(host: &Sdhci) -> [u8; 16] { mod tests { use core::ptr::NonNull; - use sdmmc_protocol::{DataDirection, cmd::cmd17}; + use sdmmc_protocol::{DataDirection, cmd::cmd17, sdio::SdioIrqHandle}; use super::*; @@ -475,20 +615,26 @@ mod tests { let mut regs = FakeRegs([0; 0x100]); let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); let mut host = unsafe { Sdhci::new(base) }; - host.irq_state - .cache(NORMAL_INT_BUFFER_WRITE_READY | NORMAL_INT_XFER_COMPLETE, 0); + host.enable_completion_irq(); + host.irq.state.begin_request(); + let generation = host.irq.state.generation(); + host.irq.state.cache_if_current( + generation, + NORMAL_INT_BUFFER_WRITE_READY | NORMAL_INT_XFER_COMPLETE, + 0, + ); let (status, _) = host.take_fifo_irq_status(NORMAL_INT_BUFFER_WRITE_READY | NORMAL_INT_ERROR); assert_ne!(status & NORMAL_INT_BUFFER_WRITE_READY, 0); assert_eq!( - host.irq_state.pending_normal() & NORMAL_INT_BUFFER_WRITE_READY, + host.irq.state.pending_normal() & NORMAL_INT_BUFFER_WRITE_READY, 0, "FIFO ready must be consumed after the data step handles it" ); assert_ne!( - host.irq_state.pending_normal() & NORMAL_INT_XFER_COMPLETE, + host.irq.state.pending_normal() & NORMAL_INT_XFER_COMPLETE, 0, "transfer completion belongs to the data-complete poll step" ); @@ -499,8 +645,12 @@ mod tests { let mut regs = FakeRegs([0; 0x100]); let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); let mut host = unsafe { Sdhci::new(base) }; - host.irq_state - .cache(NORMAL_INT_ERROR, ERROR_INT_DATA_TIMEOUT); + host.enable_completion_irq(); + host.irq.state.begin_request(); + let generation = host.irq.state.generation(); + host.irq + .state + .cache_if_current(generation, NORMAL_INT_ERROR, ERROR_INT_DATA_TIMEOUT); let (status, error) = host.take_fifo_irq_status(NORMAL_INT_BUFFER_READ_READY | NORMAL_INT_ERROR); @@ -515,8 +665,8 @@ mod tests { 0, "FIFO poll must preserve error bits after the IRQ handler clears hardware status" ); - assert_eq!(host.irq_state.pending_normal() & NORMAL_INT_ERROR, 0); - assert_eq!(host.irq_state.pending_error(), 0); + assert_eq!(host.irq.state.pending_normal() & NORMAL_INT_ERROR, 0); + assert_eq!(host.irq.state.pending_error(), 0); } #[test] @@ -524,7 +674,10 @@ mod tests { let mut regs = FakeRegs([0; 0x100]); let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); let mut host = unsafe { Sdhci::new(base) }; - host.irq_state.cache( + host.irq.state.begin_request(); + let old_generation = host.irq.state.generation(); + host.irq.state.cache_if_current( + old_generation, NORMAL_INT_CMD_COMPLETE | NORMAL_INT_XFER_COMPLETE, ERROR_INT_DATA_TIMEOUT, ); @@ -536,7 +689,52 @@ mod tests { host.submit_command(&cmd17(0)).unwrap(); - assert_eq!(host.irq_state.pending_normal(), 0); - assert_eq!(host.irq_state.pending_error(), 0); + assert_eq!(host.irq.state.pending_normal(), 0); + assert_eq!(host.irq.state.pending_error(), 0); + } + + #[test] + fn issued_command_keeps_irq_generation_active_for_completion_cache() { + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + host.enable_completion_irq(); + host.pending_data = Some(crate::host::PendingData { + direction: DataDirection::Read, + block_size: 512, + block_count: 1, + }); + + host.submit_command(&cmd17(0)).unwrap(); + assert_ne!(host.irq.state.generation(), 0); + + host.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CMD_COMPLETE); + assert_eq!( + host.irq_handle().handle_irq(), + crate::Event::CommandComplete + ); + assert_ne!( + host.irq.state.pending_normal() & NORMAL_INT_CMD_COMPLETE, + 0, + "IRQ handler must cache completion status for the active generation" + ); + } + + #[test] + fn irq_cache_drops_events_from_previous_generation() { + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let host = unsafe { Sdhci::new(base) }; + host.irq.state.begin_request(); + let old_generation = host.irq.state.generation(); + host.irq.state.end_request(); + host.irq.state.begin_request(); + assert_ne!(host.irq.state.generation(), old_generation); + + host.irq + .state + .cache_if_current(old_generation, NORMAL_INT_CMD_COMPLETE, 0); + + assert_eq!(host.irq.state.pending_normal(), 0); } } diff --git a/drivers/blk/sdhci-host/src/dma.rs b/drivers/blk/sdhci-host/src/dma.rs index 74a15fce62..bf082dae5c 100644 --- a/drivers/blk/sdhci-host/src/dma.rs +++ b/drivers/blk/sdhci-host/src/dma.rs @@ -18,9 +18,12 @@ //! bare-metal coherent systems (identity mapping, no cache ops), and //! bare-metal incoherent systems (identity mapping + dcache flush/invalidate). +use alloc::boxed::Box; use core::{num::NonZeroUsize, ptr::NonNull}; -use dma_api::{CoherentArray, DeviceDma, DmaDirection, StreamingMap}; +use dma_api::{ + CoherentArray, CompletedDma, CpuDmaBuffer, DeviceDma, DmaDirection, InFlightDma, PreparedDma, +}; use sdmmc_protocol::{ block::{ BlockPoll, BlockRequestId, BlockTransferDirection, BlockTransferMode, BlockTransferState, @@ -75,6 +78,10 @@ const ADMA2_MAX_PER_DESC: usize = 65_528; // 64 KiB - 8B, multiple of 8 pub const ADMA2_DESC_COUNT: usize = 16; pub const ADMA2_DESC_ALIGN: usize = 64; const BLOCK_SIZE: usize = 512; +pub const ADMA2_MAX_TRANSFER_SIZE: usize = + (ADMA2_DESC_COUNT * ADMA2_MAX_PER_DESC / BLOCK_SIZE) * BLOCK_SIZE; +pub const ADMA2_MAX_BLOCKS: u32 = (ADMA2_MAX_TRANSFER_SIZE / BLOCK_SIZE) as u32; +const DWC_MSHC_ADMA_BOUNDARY: u64 = 128 * 1024 * 1024; pub type RequestId = BlockRequestId; @@ -82,12 +89,37 @@ pub type RequestId = BlockRequestId; pub struct BlockRequestSlot { next: usize, state: BlockTransferState, + completed_dma: Option, +} + +impl BlockRequestSlot { + pub fn take_completed_dma(&mut self) -> Option { + self.completed_dma.take() + } } pub struct BlockRequest { inner: BlockRequestKind, } +pub struct PreparedDmaSubmitError { + pub error: Error, + buffer: Box, +} + +impl PreparedDmaSubmitError { + fn new(error: Error, buffer: PreparedDma) -> Self { + Self { + error, + buffer: Box::new(buffer), + } + } + + pub fn into_buffer(self) -> PreparedDma { + *self.buffer + } +} + // `BlockRequest` owns the DMA mappings and descriptor buffer for one // submitted transfer. Moving that ownership to another queue thread does not // grant shared access to the mapped memory; completion still requires a @@ -121,7 +153,7 @@ enum BlockRequestKind { }, Read { id: RequestId, - map: StreamingMap, + buffer: DmaRequestBuffer, _desc: CoherentArray, cmd_index: u8, phase: Phase, @@ -131,7 +163,7 @@ enum BlockRequestKind { }, Write { id: RequestId, - _map: StreamingMap, + buffer: DmaRequestBuffer, _desc: CoherentArray, cmd_index: u8, phase: Phase, @@ -141,6 +173,54 @@ enum BlockRequestKind { }, } +enum DmaRequestBuffer { + Bounce { + buffer: InFlightDma, + readback: Option<(NonNull, usize)>, + }, + Owned(InFlightDma), +} + +impl DmaRequestBuffer { + fn complete(self, read: bool) -> Option { + self.finish(read, true) + } + + fn abort(self, read: bool, quiesced: bool) -> Option { + self.finish(read, quiesced) + } + + fn finish(self, read: bool, quiesced: bool) -> Option { + match self { + Self::Bounce { buffer, readback } => { + if !quiesced { + let _quarantined = buffer.quarantine(); + return None; + } + if read { + let completed = unsafe { buffer.complete_after_quiesce() }; + if let Some((dst, len)) = readback { + completed.copy_from_device_to_slice(unsafe { + core::slice::from_raw_parts_mut(dst.as_ptr(), len) + }); + } + None + } else { + drop(unsafe { buffer.complete_after_quiesce() }); + None + } + } + Self::Owned(in_flight) => { + if !quiesced { + let _quarantined = in_flight.quarantine(); + return None; + } + Some(unsafe { in_flight.complete_after_quiesce() }) + } + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum BlockRequestStage { Command, @@ -213,10 +293,19 @@ impl BlockRequestSlot { } pub fn complete(&mut self, id: RequestId) -> Result<(), Error> { + self.complete_with_dma(id, None) + } + + fn complete_with_dma( + &mut self, + id: RequestId, + completed_dma: Option, + ) -> Result<(), Error> { if self.state.id() != Some(id) { return Err(Error::InvalidArgument); } self.state = BlockTransferState::Idle; + self.completed_dma = completed_dma; Ok(()) } @@ -255,7 +344,10 @@ pub(crate) fn build_descriptors( if written >= ADMA2_DESC_COUNT { return Err(Error::Misaligned); } - let chunk = remaining.min(ADMA2_MAX_PER_DESC); + let boundary_room = DWC_MSHC_ADMA_BOUNDARY - ((base + offset) % DWC_MSHC_ADMA_BOUNDARY); + let chunk = remaining + .min(ADMA2_MAX_PER_DESC) + .min(boundary_room as usize); let is_last = chunk == remaining; let mut attr = ADMA2_ATTR_VALID | ADMA2_ATTR_ACT_TRAN; if is_last { @@ -289,6 +381,7 @@ impl Sdhci { mode: BlockTransferMode, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let id = slot.start(mode, BlockTransferDirection::Read)?; let result = match mode { BlockTransferMode::Dma => { @@ -320,6 +413,7 @@ impl Sdhci { mode: BlockTransferMode, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let id = slot.start(mode, BlockTransferDirection::Write)?; let result = match mode { BlockTransferMode::Dma => { @@ -339,6 +433,52 @@ impl Sdhci { } } + pub fn submit_prepared_read_blocks( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + slot: &mut BlockRequestSlot, + ) -> Result { + if let Err(err) = self.check_not_poisoned() { + return Err(PreparedDmaSubmitError::new(err, buffer)); + } + let id = match slot.start(BlockTransferMode::Dma, BlockTransferDirection::Read) { + Ok(id) => id, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + match self.build_prepared_dma_read_request(start_block, buffer, dma, id) { + Ok(request) => Ok(request), + Err(err) => { + let _ = slot.complete(id); + Err(err) + } + } + } + + pub fn submit_prepared_write_blocks( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + slot: &mut BlockRequestSlot, + ) -> Result { + if let Err(err) = self.check_not_poisoned() { + return Err(PreparedDmaSubmitError::new(err, buffer)); + } + let id = match slot.start(BlockTransferMode::Dma, BlockTransferDirection::Write) { + Ok(id) => id, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + match self.build_prepared_dma_write_request(start_block, buffer, dma, id) { + Ok(request) => Ok(request), + Err(err) => { + let _ = slot.complete(id); + Err(err) + } + } + } + /// Poll a previously submitted block request. pub fn poll_block_request( &mut self, @@ -416,12 +556,11 @@ impl Sdhci { | BlockRequestKind::FifoWrite { .. } => unreachable!(), } } - return Ok(DataCommandPoll::Pending); } // Future CommandPoll variants: best-effort, treat as still pending. Ok(_) => return Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot); + let _ = self.abort_block_request(request, id, slot); return Err(err); } } @@ -437,12 +576,21 @@ impl Sdhci { // Future BlockPoll variants: best-effort, treat as still pending. Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot); + let _ = self.abort_block_request(request, id, slot); Err(err) } } } + pub fn abort_block_request_response( + &mut self, + request: &mut Option, + id: RequestId, + slot: &mut BlockRequestSlot, + ) -> Result<(), Error> { + self.abort_block_request(request, id, slot) + } + fn build_dma_read_request( &mut self, start_block: u32, @@ -455,13 +603,10 @@ impl Sdhci { return Err(Error::UnsupportedCommand); } let block_count = dma_read_block_count(size)?; - let map = dma - .map_streaming_slice_for_device( - unsafe { core::slice::from_raw_parts_mut(buffer.as_ptr(), size.get()) }, - BLOCK_SIZE, - DmaDirection::FromDevice, - ) + let backing = CpuDmaBuffer::new_zero(dma, size, BLOCK_SIZE, DmaDirection::FromDevice) .map_err(map_dma_error)?; + let dma_addr = backing.dma_addr().as_u64(); + let in_flight = unsafe { backing.prepare_for_device().into_in_flight() }; let mut desc = dma .coherent_array_zero_with_align::(ADMA2_DESC_COUNT, ADMA2_DESC_ALIGN) .map_err(map_dma_error)?; @@ -473,7 +618,7 @@ impl Sdhci { self.submit_adma2_blocks_mapped( &cmd, block_count, - map.dma_addr().as_u64(), + dma_addr, &mut desc, DataDirection::Read, Phase::DataRead, @@ -481,9 +626,12 @@ impl Sdhci { Ok(BlockRequest { inner: BlockRequestKind::Read { id, - map, + buffer: DmaRequestBuffer::Bounce { + buffer: in_flight, + readback: Some((buffer, size.get())), + }, _desc: desc, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase: Phase::DataRead, stage: BlockRequestStage::Command, stop_after_complete: block_count > 1, @@ -504,13 +652,13 @@ impl Sdhci { return Err(Error::UnsupportedCommand); } let block_count = dma_write_block_count(size)?; - let map = dma - .map_streaming_slice_for_device( - unsafe { core::slice::from_raw_parts_mut(buffer.as_ptr(), size.get()) }, - BLOCK_SIZE, - DmaDirection::ToDevice, - ) + let mut backing = CpuDmaBuffer::new_zero(dma, size, BLOCK_SIZE, DmaDirection::ToDevice) .map_err(map_dma_error)?; + backing.copy_to_device_from_slice(unsafe { + core::slice::from_raw_parts(buffer.as_ptr(), size.get()) + }); + let dma_addr = backing.dma_addr().as_u64(); + let in_flight = unsafe { backing.prepare_for_device().into_in_flight() }; let mut desc = dma .coherent_array_zero_with_align::(ADMA2_DESC_COUNT, ADMA2_DESC_ALIGN) .map_err(map_dma_error)?; @@ -522,7 +670,7 @@ impl Sdhci { self.submit_adma2_blocks_mapped( &cmd, block_count, - map.dma_addr().as_u64(), + dma_addr, &mut desc, DataDirection::Write, Phase::DataWrite, @@ -530,9 +678,126 @@ impl Sdhci { Ok(BlockRequest { inner: BlockRequestKind::Write { id, - _map: map, + buffer: DmaRequestBuffer::Bounce { + buffer: in_flight, + readback: None, + }, + _desc: desc, + cmd_index: cmd.index, + phase: Phase::DataWrite, + stage: BlockRequestStage::Command, + stop_after_complete: block_count > 1, + response: None, + }, + }) + } + + fn build_prepared_dma_read_request( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + id: RequestId, + ) -> Result { + if !self.supports_adma2() { + return Err(PreparedDmaSubmitError::new( + Error::UnsupportedCommand, + buffer, + )); + } + if buffer.direction() != DmaDirection::FromDevice || buffer.domain_id() != dma.domain_id() { + return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)); + } + let block_count = match dma_read_block_count(buffer.len()) { + Ok(block_count) => block_count, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + let mut desc = match dma + .coherent_array_zero_with_align::(ADMA2_DESC_COUNT, ADMA2_DESC_ALIGN) + { + Ok(desc) => desc, + Err(err) => return Err(PreparedDmaSubmitError::new(map_dma_error(err), buffer)), + }; + let cmd = if block_count == 1 { + cmd17(start_block) + } else { + cmd18(start_block) + }; + match self.submit_adma2_blocks_mapped( + &cmd, + block_count, + buffer.dma_addr().as_u64(), + &mut desc, + DataDirection::Read, + Phase::DataRead, + ) { + Ok(()) => {} + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + } + let buffer = unsafe { buffer.into_in_flight() }; + Ok(BlockRequest { + inner: BlockRequestKind::Read { + id, + buffer: DmaRequestBuffer::Owned(buffer), + _desc: desc, + cmd_index: cmd.index, + phase: Phase::DataRead, + stage: BlockRequestStage::Command, + stop_after_complete: block_count > 1, + response: None, + }, + }) + } + + fn build_prepared_dma_write_request( + &mut self, + start_block: u32, + buffer: PreparedDma, + dma: &DeviceDma, + id: RequestId, + ) -> Result { + if !self.supports_adma2() { + return Err(PreparedDmaSubmitError::new( + Error::UnsupportedCommand, + buffer, + )); + } + if buffer.direction() != DmaDirection::ToDevice || buffer.domain_id() != dma.domain_id() { + return Err(PreparedDmaSubmitError::new(Error::InvalidArgument, buffer)); + } + let block_count = match dma_write_block_count(buffer.len()) { + Ok(block_count) => block_count, + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + }; + let mut desc = match dma + .coherent_array_zero_with_align::(ADMA2_DESC_COUNT, ADMA2_DESC_ALIGN) + { + Ok(desc) => desc, + Err(err) => return Err(PreparedDmaSubmitError::new(map_dma_error(err), buffer)), + }; + let cmd = if block_count == 1 { + cmd24(start_block) + } else { + cmd25(start_block) + }; + match self.submit_adma2_blocks_mapped( + &cmd, + block_count, + buffer.dma_addr().as_u64(), + &mut desc, + DataDirection::Write, + Phase::DataWrite, + ) { + Ok(()) => {} + Err(err) => return Err(PreparedDmaSubmitError::new(err, buffer)), + } + let buffer = unsafe { buffer.into_in_flight() }; + Ok(BlockRequest { + inner: BlockRequestKind::Write { + id, + buffer: DmaRequestBuffer::Owned(buffer), _desc: desc, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase: Phase::DataWrite, stage: BlockRequestStage::Command, stop_after_complete: block_count > 1, @@ -602,6 +867,7 @@ impl Sdhci { direction: DataDirection, slot: &mut BlockRequestSlot, ) -> Result { + self.check_not_poisoned()?; let transfer_direction = match direction { DataDirection::Read => BlockTransferDirection::Read, DataDirection::Write => BlockTransferDirection::Write, @@ -669,7 +935,7 @@ impl Sdhci { len, block_size: block_size_usize, offset: 0, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase, stage: BlockRequestStage::Command, stop_after_complete, @@ -681,7 +947,7 @@ impl Sdhci { len, block_size: block_size_usize, offset: 0, - cmd_index: cmd.cmd, + cmd_index: cmd.index, phase, stage: BlockRequestStage::Command, stop_after_complete, @@ -732,23 +998,53 @@ impl Sdhci { response } - fn finish_block_request(&mut self, request: BlockRequest) -> Result<(), Error> { - match request.inner { - BlockRequestKind::FifoRead { .. } | BlockRequestKind::FifoWrite { .. } => {} - BlockRequestKind::Read { stage, .. } => { + fn finish_block_request( + &mut self, + request: BlockRequest, + ) -> Result, Error> { + self.finish_block_request_with_quiesce(request, true) + } + + fn finish_block_request_with_quiesce( + &mut self, + request: BlockRequest, + quiesced: bool, + ) -> Result, Error> { + if !quiesced { + self.poison_dma(); + core::mem::forget(request); + self.pending_data = None; + self.active_data_cmd = 0; + self.irq.state.end_request(); + return Ok(None); + } + let completed_dma = match request.inner { + BlockRequestKind::FifoRead { .. } | BlockRequestKind::FifoWrite { .. } => None, + BlockRequestKind::Read { stage, buffer, .. } => { if stage == BlockRequestStage::Command { let _ = self.take_command_response(); } + if quiesced { + buffer.complete(true) + } else { + buffer.abort(true, false) + } } - BlockRequestKind::Write { stage, .. } => { + BlockRequestKind::Write { stage, buffer, .. } => { if stage == BlockRequestStage::Command { let _ = self.take_command_response(); } + if quiesced { + buffer.complete(false) + } else { + buffer.abort(false, false) + } } - } + }; self.pending_data = None; self.active_data_cmd = 0; - Ok(()) + self.irq.state.end_request(); + Ok(completed_dma) } fn finish_dma_data( @@ -763,12 +1059,10 @@ impl Sdhci { let stop_after_complete = match &mut active.inner { BlockRequestKind::Read { - map, stop_after_complete, stage, .. } => { - map.complete_for_cpu_all(); *stage = BlockRequestStage::Stop; *stop_after_complete } @@ -792,8 +1086,8 @@ impl Sdhci { let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active)?; - slot.complete(id)?; + let completed_dma = self.finish_block_request(active)?; + slot.complete_with_dma(id, completed_dma)?; Ok(DataCommandPoll::Complete(response)) } @@ -809,14 +1103,14 @@ impl Sdhci { let _ = self.take_command_response()?; let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active)?; - slot.complete(id)?; + let completed_dma = self.finish_block_request(active)?; + slot.complete_with_dma(id, completed_dma)?; Ok(DataCommandPoll::Complete(response)) } // Future CommandPoll variants: best-effort, treat as still pending. Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot); + let _ = self.abort_block_request(request, id, slot); Err(err) } } @@ -863,12 +1157,11 @@ impl Sdhci { } } set_fifo_stage(request, BlockRequestStage::Data)?; - return Ok(DataCommandPoll::Pending); } // Future CommandPoll variants: best-effort, treat as still pending. Ok(_) => return Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot); + let _ = self.abort_block_request(request, id, slot); return Err(err); } } @@ -890,7 +1183,7 @@ impl Sdhci { // Future BlockPoll variants: best-effort, treat as still pending. Ok(_) => Ok(DataCommandPoll::Pending), Err(err) => { - self.abort_block_request(request, id, slot); + let _ = self.abort_block_request(request, id, slot); Err(err) } } @@ -957,7 +1250,8 @@ impl Sdhci { let active = request.take().ok_or(Error::InvalidArgument)?; let response = active.response().ok_or(Error::InvalidArgument)?; - self.finish_block_request(active)?; + let completed_dma = self.finish_block_request(active)?; + drop(completed_dma); slot.complete(id)?; Ok(DataCommandPoll::Complete(response)) } @@ -967,21 +1261,38 @@ impl Sdhci { request: &mut Option, id: RequestId, slot: &mut BlockRequestSlot, - ) { - let _ = request.take(); - self.recover_after_adma2_error(); - let _ = slot.complete(id); + ) -> Result<(), Error> { + let active = request.take().ok_or(Error::InvalidArgument)?; + let recovery = self.recover_after_adma2_error(); + let completed_dma = self.finish_block_request_with_quiesce(active, recovery.is_ok())?; + drop(completed_dma); + slot.complete(id)?; + recovery } - fn recover_after_adma2_error(&mut self) { + fn recover_after_adma2_error(&mut self) -> Result<(), Error> { + let was_irq_enabled = self.completion_irq_enabled(); self.use_dma = false; self.pending_data = None; self.active_data_cmd = 0; self.command_state = CommandState::Idle; self.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CLEAR_ALL); self.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_CLEAR_ALL); - let _ = self.reset_cmd(); - let _ = self.reset_dat(); + self.clear_cached_irq_status(); + + let cmd = self.reset_cmd(); + let dat = self.reset_dat(); + match (cmd, dat) { + (Ok(()), Ok(())) => Ok(()), + (Err(err), _) | (_, Err(err)) => { + let fallback = self.reset_all(); + self.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CLEAR_ALL); + self.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_CLEAR_ALL); + self.clear_cached_irq_status(); + self.restore_completion_irq_after_reset(was_irq_enabled); + fallback.map_err(|_| err) + } + } } pub(crate) fn poll_data_complete_with_adma( @@ -1009,6 +1320,24 @@ impl Sdhci { } Ok(BlockPoll::Pending) } + + fn poll_fifo_data_complete( + &mut self, + cmd_index: u8, + phase: Phase, + write: bool, + ) -> Result { + match self.poll_data_complete_with_adma(cmd_index, phase)? { + BlockPoll::Pending if !data_line_inhibited(self) => Ok(BlockPoll::Complete), + // Some DWCMSHC instances can miss the polling-visible transfer + // complete bit for PIO writes. Once the FIFO path has pushed the + // last word, DAT0 high is the card-side busy release signal; the + // buffer-write-ready bit is not guaranteed to remain asserted at + // that point. + BlockPoll::Pending if write && fifo_write_not_busy(self) => Ok(BlockPoll::Complete), + poll => Ok(poll), + } + } } fn build_descriptors_into_dma( @@ -1054,12 +1383,17 @@ fn poll_fifo_read_step( phase: Phase, ) -> Result { if *offset >= len { - return host.poll_data_complete_with_adma(cmd_index, phase); + return host.poll_fifo_data_complete(cmd_index, phase, false); } let (status, error) = host.take_fifo_irq_status(NORMAL_INT_BUFFER_READ_READY | NORMAL_INT_ERROR); - if status & NORMAL_INT_BUFFER_READ_READY == 0 { + if status & NORMAL_INT_ERROR != 0 { + return poll_fifo_status(host, status, error, cmd_index, phase, true); + } + if status & NORMAL_INT_BUFFER_READ_READY == 0 + && !fifo_present_state_ready(host, PRESENT_BUFFER_READ_ENABLE) + { return poll_fifo_status(host, status, error, cmd_index, phase, true); } @@ -1087,12 +1421,17 @@ fn poll_fifo_write_step( phase: Phase, ) -> Result { if *offset >= len { - return host.poll_data_complete_with_adma(cmd_index, phase); + return host.poll_fifo_data_complete(cmd_index, phase, true); } let (status, error) = host.take_fifo_irq_status(NORMAL_INT_BUFFER_WRITE_READY | NORMAL_INT_ERROR); - if status & NORMAL_INT_BUFFER_WRITE_READY == 0 { + if status & NORMAL_INT_ERROR != 0 { + return poll_fifo_status(host, status, error, cmd_index, phase, false); + } + if status & NORMAL_INT_BUFFER_WRITE_READY == 0 + && !fifo_present_state_ready(host, PRESENT_BUFFER_WRITE_ENABLE) + { return poll_fifo_status(host, status, error, cmd_index, phase, false); } @@ -1109,6 +1448,18 @@ fn poll_fifo_write_step( Ok(BlockPoll::Pending) } +fn fifo_present_state_ready(host: &Sdhci, ready_mask: u32) -> bool { + host.read_u32(REG_PRESENT_STATE) & ready_mask != 0 +} + +fn data_line_inhibited(host: &Sdhci) -> bool { + host.read_u32(REG_PRESENT_STATE) & PRESENT_DAT_INHIBIT != 0 +} + +fn fifo_write_not_busy(host: &Sdhci) -> bool { + host.read_u32(REG_PRESENT_STATE) & PRESENT_DAT0_LINE_SIGNAL_LEVEL != 0 +} + fn poll_fifo_status( host: &mut Sdhci, status: u16, @@ -1174,8 +1525,15 @@ fn map_dma_error(err: dma_api::DmaError) -> Error { #[cfg(test)] mod tests { + use core::ptr::NonNull; + + use sdmmc_protocol::response::Response; + use super::*; + #[repr(align(4))] + struct FakeRegs([u8; 0x100]); + fn empty_table() -> [Adma2Desc32; ADMA2_DESC_COUNT] { [Adma2Desc32 { attr: 0, @@ -1213,6 +1571,21 @@ mod tests { assert_eq!(table[1].address, 0x2000_0000 + ADMA2_MAX_PER_DESC as u32); } + #[test] + fn splits_at_dwcmshc_128m_boundary() { + let mut table = empty_table(); + let base = DWC_MSHC_ADMA_BOUNDARY - 1024; + let n = build_descriptors(&mut table, base, 4096, Phase::DataRead).unwrap(); + + assert_eq!(n, 2); + assert_eq!(table[0].length, 1024); + assert_eq!(table[0].address, base as u32); + assert!(table[0].attr & ADMA2_ATTR_END == 0); + assert_eq!(table[1].length, 3072); + assert_eq!(table[1].address, DWC_MSHC_ADMA_BOUNDARY as u32); + assert!(table[1].attr & ADMA2_ATTR_END != 0); + } + #[test] fn rejects_64bit_bus_address() { let mut table = empty_table(); @@ -1274,4 +1647,179 @@ mod tests { assert_send::(); assert_send::(); } + + #[test] + fn block_poll_consumes_data_complete_cached_with_command_complete() { + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + let mut slot = BlockRequestSlot::default(); + let id = slot + .start(BlockTransferMode::Fifo, BlockTransferDirection::Write) + .unwrap(); + let buffer = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut request = Some(BlockRequest { + inner: BlockRequestKind::FifoWrite { + id, + buffer, + len: 0, + block_size: BLOCK_SIZE, + offset: 0, + cmd_index: 24, + phase: Phase::DataWrite, + stage: BlockRequestStage::Command, + stop_after_complete: false, + response: None, + }, + }); + host.command_state = CommandState::Complete { + response: Response::Empty, + }; + host.enable_completion_irq(); + host.irq.state.begin_request(); + let generation = host.irq.state.generation(); + host.irq.state.cache_if_current( + generation, + NORMAL_INT_CMD_COMPLETE | NORMAL_INT_XFER_COMPLETE, + 0, + ); + + assert_eq!( + host.poll_block_request(&mut request, id, &mut slot), + Ok(BlockPoll::Complete) + ); + assert!(request.is_none()); + assert!(matches!(slot.state(), BlockTransferState::Idle)); + } + + #[test] + fn fifo_write_step_accepts_present_state_ready_without_irq_status() { + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + let mut buffer = [0x5au8; BLOCK_SIZE]; + buffer[BLOCK_SIZE - 4..].copy_from_slice(&0x1122_3344u32.to_le_bytes()); + let ptr = NonNull::new(buffer.as_mut_ptr()).unwrap(); + let mut offset = 0; + host.write_u32(REG_PRESENT_STATE, PRESENT_BUFFER_WRITE_ENABLE); + + assert_eq!( + poll_fifo_write_step( + &mut host, + ptr, + buffer.len(), + BLOCK_SIZE, + &mut offset, + 24, + Phase::DataWrite, + ), + Ok(BlockPoll::Pending) + ); + + assert_eq!(offset, BLOCK_SIZE); + assert_eq!(host.read_u32(REG_BUFFER_DATA_PORT), 0x1122_3344); + } + + #[test] + fn fifo_read_step_accepts_present_state_ready_without_irq_status() { + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + let mut buffer = [0u8; BLOCK_SIZE]; + let ptr = NonNull::new(buffer.as_mut_ptr()).unwrap(); + let mut offset = 0; + host.write_u32(REG_PRESENT_STATE, PRESENT_BUFFER_READ_ENABLE); + host.write_u32(REG_BUFFER_DATA_PORT, 0xaabb_ccdd); + + assert_eq!( + poll_fifo_read_step( + &mut host, + ptr, + 4, + BLOCK_SIZE, + &mut offset, + 17, + Phase::DataRead, + ), + Ok(BlockPoll::Pending) + ); + + assert_eq!(offset, 4); + assert_eq!(&buffer[..4], &0xaabb_ccddu32.to_le_bytes()); + } + + #[test] + fn fifo_data_complete_accepts_dat_inhibit_clear_without_irq_status() { + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + let mut buffer = [0u8; BLOCK_SIZE]; + let ptr = NonNull::new(buffer.as_mut_ptr()).unwrap(); + let mut offset = BLOCK_SIZE; + host.write_u32(REG_PRESENT_STATE, 0); + + assert_eq!( + poll_fifo_read_step( + &mut host, + ptr, + BLOCK_SIZE, + BLOCK_SIZE, + &mut offset, + 17, + Phase::DataRead, + ), + Ok(BlockPoll::Complete) + ); + } + + #[test] + fn fifo_write_complete_waits_while_dat0_busy_without_xfer_irq() { + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + let mut buffer = [0u8; BLOCK_SIZE]; + let ptr = NonNull::new(buffer.as_mut_ptr()).unwrap(); + let mut offset = BLOCK_SIZE; + host.write_u32(REG_PRESENT_STATE, PRESENT_DAT_INHIBIT); + + assert_eq!( + poll_fifo_write_step( + &mut host, + ptr, + BLOCK_SIZE, + BLOCK_SIZE, + &mut offset, + 24, + Phase::DataWrite, + ), + Ok(BlockPoll::Pending) + ); + } + + #[test] + fn fifo_write_complete_accepts_dat0_ready_without_xfer_irq_or_write_ready() { + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + let mut buffer = [0u8; BLOCK_SIZE]; + let ptr = NonNull::new(buffer.as_mut_ptr()).unwrap(); + let mut offset = BLOCK_SIZE; + host.write_u32( + REG_PRESENT_STATE, + PRESENT_DAT_INHIBIT | PRESENT_DAT0_LINE_SIGNAL_LEVEL, + ); + + assert_eq!( + poll_fifo_write_step( + &mut host, + ptr, + BLOCK_SIZE, + BLOCK_SIZE, + &mut offset, + 24, + Phase::DataWrite, + ), + Ok(BlockPoll::Complete) + ); + } } diff --git a/drivers/blk/sdhci-host/src/host.rs b/drivers/blk/sdhci-host/src/host.rs index e33930dd9c..8b95161f1d 100644 --- a/drivers/blk/sdhci-host/src/host.rs +++ b/drivers/blk/sdhci-host/src/host.rs @@ -1,8 +1,9 @@ //! `Sdhci` core: MMIO accessors, reset, clock and bus-width setup. +use alloc::sync::Arc; use core::{ ptr::NonNull, - sync::atomic::{AtomicU32, Ordering}, + sync::atomic::{AtomicU32, AtomicU64, Ordering}, }; use dma_api::DeviceDma; @@ -32,68 +33,188 @@ pub(crate) struct PendingData { /// `new` is `unsafe` because the caller must provide a valid, exclusive /// MMIO base address for an SDHCI v3.x compatible controller. Concurrent /// use of the same controller from multiple `Sdhci` instances is undefined. +const IRQ_GENERATION_SHIFT: u64 = 32; +const IRQ_NORMAL_MASK: u64 = 0xffff; +const IRQ_ERROR_SHIFT: u64 = 16; + pub(crate) struct IrqState { - pending_normal: AtomicU32, - pending_error: AtomicU32, + mailbox: AtomicU64, + next_generation: AtomicU32, } impl IrqState { const fn new() -> Self { Self { - pending_normal: AtomicU32::new(0), - pending_error: AtomicU32::new(0), + mailbox: AtomicU64::new(0), + next_generation: AtomicU32::new(0), } } - pub(crate) fn cache(&self, normal: u16, error: u16) { - if normal != 0 { - self.pending_normal - .fetch_or(normal as u32, Ordering::AcqRel); + pub(crate) fn begin_request(&self) { + let generation = self.next_generation(); + self.mailbox + .store(pack_mailbox(generation, 0, 0), Ordering::Release); + } + + pub(crate) fn end_request(&self) { + self.mailbox.store(0, Ordering::Release); + } + + pub(crate) fn cache_if_current(&self, generation: u32, normal: u16, error: u16) { + if generation == 0 || (normal == 0 && error == 0) { + return; } - if error != 0 { - self.pending_error.fetch_or(error as u32, Ordering::AcqRel); + let mut cur = self.mailbox.load(Ordering::Acquire); + loop { + if mailbox_generation(cur) != generation { + return; + } + let next = pack_mailbox( + generation, + mailbox_normal(cur) | normal, + mailbox_error(cur) | error, + ); + match self + .mailbox + .compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return, + Err(observed) => cur = observed, + } } } + pub(crate) fn generation(&self) -> u32 { + mailbox_generation(self.mailbox.load(Ordering::Acquire)) + } + pub(crate) fn take_normal(&self, mask: u16) -> u16 { - take_cached_bits(&self.pending_normal, mask as u32) as u16 + let mut cur = self.mailbox.load(Ordering::Acquire); + loop { + let normal = mailbox_normal(cur); + let taken = normal & mask; + if taken == 0 { + return 0; + } + let next = pack_mailbox(mailbox_generation(cur), normal & !mask, mailbox_error(cur)); + match self + .mailbox + .compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return taken, + Err(observed) => cur = observed, + } + } } pub(crate) fn take_error_all(&self) -> u16 { - self.pending_error.swap(0, Ordering::AcqRel) as u16 + let mut cur = self.mailbox.load(Ordering::Acquire); + loop { + let error = mailbox_error(cur); + if error == 0 { + return 0; + } + let next = pack_mailbox(mailbox_generation(cur), mailbox_normal(cur), 0); + match self + .mailbox + .compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return error, + Err(observed) => cur = observed, + } + } } pub(crate) fn clear_normal(&self, mask: u16) { - self.pending_normal - .fetch_and(!(mask as u32), Ordering::AcqRel); + let mut cur = self.mailbox.load(Ordering::Acquire); + loop { + let next = pack_mailbox( + mailbox_generation(cur), + mailbox_normal(cur) & !mask, + mailbox_error(cur), + ); + match self + .mailbox + .compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return, + Err(observed) => cur = observed, + } + } } pub(crate) fn clear_all(&self) { - self.pending_normal.store(0, Ordering::Release); - self.pending_error.store(0, Ordering::Release); + let mut cur = self.mailbox.load(Ordering::Acquire); + loop { + let next = pack_mailbox(mailbox_generation(cur), 0, 0); + match self + .mailbox + .compare_exchange_weak(cur, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return, + Err(observed) => cur = observed, + } + } } #[cfg(test)] pub(crate) fn pending_normal(&self) -> u16 { - self.pending_normal.load(Ordering::Acquire) as u16 + mailbox_normal(self.mailbox.load(Ordering::Acquire)) } #[cfg(test)] pub(crate) fn pending_error(&self) -> u16 { - self.pending_error.load(Ordering::Acquire) as u16 + mailbox_error(self.mailbox.load(Ordering::Acquire)) } -} -fn take_cached_bits(cache: &AtomicU32, mask: u32) -> u32 { - let mut cur = cache.load(Ordering::Acquire); - loop { - let taken = cur & mask; - if taken == 0 { - return 0; + fn next_generation(&self) -> u32 { + let mut cur = self.next_generation.load(Ordering::Acquire); + loop { + let mut next = cur.wrapping_add(1); + if next == 0 { + next = 1; + } + match self.next_generation.compare_exchange_weak( + cur, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return next, + Err(observed) => cur = observed, + } } - match cache.compare_exchange_weak(cur, cur & !mask, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => return taken, - Err(next) => cur = next, + } +} + +fn pack_mailbox(generation: u32, normal: u16, error: u16) -> u64 { + ((generation as u64) << IRQ_GENERATION_SHIFT) + | normal as u64 + | ((error as u64) << IRQ_ERROR_SHIFT) +} + +fn mailbox_generation(value: u64) -> u32 { + (value >> IRQ_GENERATION_SHIFT) as u32 +} + +fn mailbox_normal(value: u64) -> u16 { + (value & IRQ_NORMAL_MASK) as u16 +} + +fn mailbox_error(value: u64) -> u16 { + ((value >> IRQ_ERROR_SHIFT) & IRQ_NORMAL_MASK) as u16 +} + +pub(crate) struct IrqCore { + pub(crate) base_addr: usize, + pub(crate) state: IrqState, +} + +impl IrqCore { + fn new(base_addr: usize) -> Self { + Self { + base_addr, + state: IrqState::new(), } } } @@ -111,6 +232,14 @@ pub struct Sdhci { /// for 1:1 passthrough) instead of using the internal 10-bit divider. /// Used on controllers whose internal divider is unusable. pub(crate) ext_clock: Option<&'static dyn HostClock>, + /// Optional platform hook that runs after a controller-wide reset has + /// completed and before protocol commands are issued. DWCMSHC-style + /// integrations use this for vendor PHY/DLL defaults that reset does not + /// leave in a usable identification-mode state. + pub(crate) reset_hook: Option<&'static dyn HostResetHook>, + /// Optional monotonic timer used by asynchronous bus-operation state + /// machines that have specification-defined wall-clock delays. + pub(crate) timer: Option<&'static dyn HostTimer>, /// Whether the platform has wired up the IO-domain regulator needed to /// actually run the bus at 1.8 V. Default `false` — toggling /// `HOST_CONTROL2.1V8_SIGNALING_ENABLE` alone changes the controller @@ -123,7 +252,10 @@ pub struct Sdhci { pub(crate) active_data_cmd: u8, pub(crate) dma: Option, pub(crate) dma_mask: u64, - pub(crate) irq_state: IrqState, + pub(crate) dma_poisoned: bool, + pub(crate) irq: Arc, + pub(crate) host2_next_id: u64, + pub(crate) host2_active_id: Option, } impl Sdhci { @@ -140,11 +272,16 @@ impl Sdhci { pending_data: None, use_dma: false, ext_clock: None, + reset_hook: None, + timer: None, support_1v8: false, active_data_cmd: 0, dma: None, dma_mask: u32::MAX as u64, - irq_state: IrqState::new(), + dma_poisoned: false, + irq: Arc::new(IrqCore::new(base.as_ptr() as usize)), + host2_next_id: 0, + host2_active_id: None, } } @@ -175,6 +312,11 @@ impl Sdhci { unsafe { Self::new(base) } } + /// Return the mapped MMIO base address owned by this driver instance. + pub fn mmio_base(&self) -> usize { + self.base_addr + } + /// Install a CRU-side clock callback so subsequent `set_clock` calls /// retune the platform's reference clock instead of using the SDHCI /// internal divider. The callback receives the desired SD bus @@ -191,6 +333,24 @@ impl Sdhci { self.ext_clock = Some(clock); } + /// Install a platform post-reset hook. The hook is called after ResetAll + /// clears, both for the legacy blocking reset helper and for the native + /// `sdio-host2` bus-operation state machine. + pub fn set_reset_hook(&mut self, hook: &'static H) + where + H: HostResetHook + 'static, + { + self.reset_hook = Some(hook); + } + + /// Install a platform monotonic timer in milliseconds. + pub fn set_timer(&mut self, timer: &'static T) + where + T: HostTimer + 'static, + { + self.timer = Some(timer); + } + /// Declare that the platform can switch the SD/eMMC IO rail to 1.8 V. /// /// Until this is called, [`SdioHost::switch_voltage`] refuses @@ -213,10 +373,23 @@ impl Sdhci { self.dma = Some(dma); } + pub(crate) fn check_not_poisoned(&self) -> Result<(), Error> { + if self.dma_poisoned { + Err(Error::BusError(ErrorContext::new(Phase::DataRead))) + } else { + Ok(()) + } + } + + pub(crate) fn poison_dma(&mut self) { + self.dma_poisoned = true; + } + /// Reset the controller (CMD line + DAT line + state) by writing the /// "Reset All" bit and waiting for it to clear. pub fn reset_all(&mut self) -> Result<(), Error> { self.reset_with_mask(RESET_ALL, Phase::Init) + .inspect(|_| self.dma_poisoned = false) } /// Reset the CMD line state machine (clears any stuck CMD inhibit). @@ -233,6 +406,11 @@ impl Sdhci { self.write_u8(REG_SOFTWARE_RESET, mask); for _ in 0..1000 { if self.read_u8(REG_SOFTWARE_RESET) & mask == 0 { + if mask == RESET_ALL + && let Some(hook) = self.reset_hook + { + hook.after_reset(self)?; + } return Ok(()); } spin_loop(); @@ -435,6 +613,17 @@ pub trait HostClock: Sync { fn set_clock(&self, target_hz: u32) -> Result<(), Error>; } +/// Platform hook for SDHCI integrations that need vendor register setup after +/// controller ResetAll has completed. +pub trait HostResetHook: Sync { + fn after_reset(&self, host: &mut Sdhci) -> Result<(), Error>; +} + +/// Platform monotonic-time capability used for specification-defined delays. +pub trait HostTimer: Sync { + fn now_ms(&self) -> u64; +} + #[inline] fn spin_loop() { core::hint::spin_loop(); diff --git a/drivers/blk/sdhci-host/src/lib.rs b/drivers/blk/sdhci-host/src/lib.rs index 1e6e9ece52..0f5392e31d 100644 --- a/drivers/blk/sdhci-host/src/lib.rs +++ b/drivers/blk/sdhci-host/src/lib.rs @@ -1,17 +1,19 @@ //! SDHCI host controller backend for the `sdmmc-protocol` driver crate. //! //! This crate ports the [SD Host Controller Standard Specification][sdhci] -//! v3.x register layout and PIO data path into a [`SdioHost`] implementation -//! that the [`sdmmc_protocol::sdio::SdioSdmmc`] driver can drive directly. +//! v3.x register layout and PIO data path into a physical +//! [`sdio_host2::SdioHost`] implementation that +//! [`sdmmc_protocol::sdio::SdioSdmmc`] drives through +//! [`sdmmc_protocol::sdio::SdioSdmmc::new_host2`]. //! //! # Scope //! //! - **Implemented**: PIO transfers, **ADMA2 (32-bit) transfers**, 1-bit / -//! 4-bit bus, default-speed and high-speed clocking, 32-bit response +//! 4-bit / 8-bit bus, default-speed and high-speed clocking, 32-bit response //! slots, 136-bit R2 reconstruction, software reset / clock setup. -//! - **Out of scope (for now)**: 64-bit ADMA2, 8-bit eMMC bus, HS200 / -//! SDR50 / SDR104 clocking, tuning (CMD19 / CMD21), eMMC-specific -//! commands. 1.8 V signaling is wired up at the register level but is +//! - **Out of scope (for now)**: 64-bit ADMA2, HS200 / SDR50 / SDR104 +//! clocking, tuning (CMD19 / CMD21), eMMC-specific commands beyond normal +//! block I/O. 1.8 V signaling is wired up at the register level but is //! gated behind [`Sdhci::enable_1v8_signaling`] — platforms that haven't //! plumbed the IO-rail regulator MUST leave it off so the protocol //! layer falls back instead of corrupting transfers. @@ -26,18 +28,19 @@ //! //! let mmio = NonNull::new(0xFE31_0000 as *mut u8).unwrap(); //! let host = unsafe { Sdhci::new(mmio) }; -//! let mut card = SdioSdmmc::new(host); +//! let mut card = SdioSdmmc::new_host2(host); //! let mut scratch = SdioInitScratch::new(); //! let mut request = card.submit_init(&mut scratch)?; //! // Poll request here. Runtime code chooses spin, yield, IRQ wait, or timer. //! # Ok::<(), sdmmc_protocol::Error>(()) //! ``` //! -//! For block request I/O, use [`Sdhci::submit_read_blocks`] or -//! [`Sdhci::submit_write_blocks`] and complete the returned request with -//! [`Sdhci::poll_block_request`]. `BlockTransferMode::Dma` maps the request -//! buffer and builds the ADMA2 descriptor table; `BlockTransferMode::Fifo` -//! uses the controller FIFO with the same submit/poll contract: +//! Low-level block request primitives remain available for controller bring-up +//! and diagnostics. Normal block-device users should prefer [`rdif::device`], +//! which routes RDIF requests through the shared SD/MMC protocol state machine +//! and this host's native `sdio-host2` transaction path. The raw primitives use +//! [`Sdhci::submit_read_blocks`] or [`Sdhci::submit_write_blocks`] and complete +//! the returned request with [`Sdhci::poll_block_request`]: //! //! ```ignore //! use core::{num::NonZeroUsize, ptr::NonNull}; @@ -45,7 +48,7 @@ //! use sdhci_host::{BlockRequestSlot, BlockTransferMode, RequestId, Sdhci}; //! //! # use platform::DmaImpl; -//! let dma = DeviceDma::new(u32::MAX as u64, &DmaImpl); +//! let dma = DeviceDma::new_legacy(u32::MAX as u64, &DmaImpl); //! let mut host = unsafe { Sdhci::new_from_addr(0xFE31_0000) }; //! let mut block = [0u8; 512]; //! let ptr = NonNull::new(block.as_mut_ptr()).unwrap(); @@ -71,26 +74,39 @@ #![no_std] #![allow(clippy::missing_safety_doc)] -use core::{marker::PhantomData, num::NonZeroUsize, ptr::NonNull}; +extern crate alloc; + +use alloc::sync::Arc; +use core::{ + marker::PhantomData, + num::NonZeroUsize, + ptr::NonNull, + sync::atomic::{AtomicBool, Ordering}, +}; mod command; mod dma; mod host; +pub mod rdif; mod regs; -pub use dma::{ADMA2_DESC_ALIGN, ADMA2_DESC_COUNT, BlockRequest, BlockRequestSlot, RequestId}; -pub use host::{HostClock, Sdhci}; +pub use dma::{ + ADMA2_DESC_ALIGN, ADMA2_DESC_COUNT, ADMA2_MAX_BLOCKS, ADMA2_MAX_TRANSFER_SIZE, BlockRequest, + BlockRequestSlot, RequestId, +}; +pub use host::{HostClock, HostResetHook, HostTimer, Sdhci}; pub use sdmmc_protocol::block::{ BlockBufferConfig, BlockPoll, BlockRequestId, BlockTransferDirection, BlockTransferMode, BlockTransferState, }; use sdmmc_protocol::{ - DataCommandPoll, + DataCommandPoll, OperationPoll, cmd::{Command, DataDirection}, error::{Error, ErrorContext, Phase}, sdio::{ - BusWidth, ClockSpeed, HostEvent, HostEventKind, HostEventSource, SdioHost, SdioIrqHandle, - SdioIrqHost, SignalVoltage, + BusWidth, ClockSpeed, HostEvent, HostEventKind, HostEventSource, ReadyBusRequest, + SdioBusOp, SdioHost as ProtocolSdioHost, SdioIrqHandle, SdioIrqHost, SignalVoltage, + poll_ready_bus_op, submit_ready_bus_op, }, }; @@ -106,6 +122,10 @@ pub enum Event { CommandComplete, /// A data transfer has completed. TransferComplete, + /// Receive-side FIFO data is ready. + ReceiveReady, + /// Transmit-side FIFO space is ready. + TransmitReady, /// One or more error bits are pending. Error { normal: u16, error: u16 }, /// Status bits are pending but do not map to a high-level event yet. @@ -119,24 +139,136 @@ pub struct DataRequest<'a> { _buffer: PhantomData<&'a [u8]>, } +pub struct TransactionRequest<'a> { + owner: usize, + id: u64, + done: bool, + kind: TransactionRequestKind, + data: Option>, +} + +static ADMA_READ_PATH_LOGGED: AtomicBool = AtomicBool::new(false); +static ADMA_WRITE_PATH_LOGGED: AtomicBool = AtomicBool::new(false); +static ADMA_READ_FALLBACK_LOGGED: AtomicBool = AtomicBool::new(false); +static ADMA_WRITE_FALLBACK_LOGGED: AtomicBool = AtomicBool::new(false); + +enum TransactionRequestKind { + Command { response: sdio_host2::ResponseType }, + Data { response: sdio_host2::ResponseType }, +} + +impl<'a> TransactionRequest<'a> { + fn command(owner: usize, id: u64, response: sdio_host2::ResponseType) -> Self { + Self { + owner, + id, + done: false, + kind: TransactionRequestKind::Command { response }, + data: None, + } + } + + fn data( + owner: usize, + id: u64, + request: DataRequest<'a>, + response: sdio_host2::ResponseType, + ) -> Self { + Self { + owner, + id, + done: false, + kind: TransactionRequestKind::Data { response }, + data: Some(request), + } + } +} + +pub struct BusRequest { + owner: usize, + id: u64, + done: bool, + state: BusRequestState, +} + +impl BusRequest { + fn pending(owner: usize, id: u64, state: BusRequestState) -> Self { + Self { + owner, + id, + done: false, + state, + } + } +} + +enum BusRequestState { + Reset { + mask: u8, + phase: Phase, + was_irq_enabled: bool, + started: bool, + polls: u32, + }, + PowerOn, + PowerOff, + SetClock(SdhciClockState), + SetBusWidth(BusWidth), + SetSignalVoltage(SdhciVoltageState), + ExecuteTuning(SdhciTuningState), +} + +enum SdhciClockState { + Start { + target_hz: u32, + uhs_mode: Option, + high_speed: Option, + }, + ExternalSetClock { + target_hz: u32, + }, + ExternalEnable { + polls: u32, + }, + InternalWaitStable { + polls: u32, + }, +} + +enum SdhciVoltageState { + DisableClock(SignalVoltage), + SwitchControllerAndRail(SignalVoltage), + WaitVsw { + voltage: SignalVoltage, + deadline_ms: Option, + }, + EnableClock(SignalVoltage), + VerifyDatLines(SignalVoltage), +} + +enum SdhciTuningState { + Start { cmd_index: u8, block_size: u16 }, + Wait { cmd_index: u8, polls: u32 }, +} + +const SDHCI_RESET_POLLS: u32 = 1_000; +const SDHCI_CLOCK_POLLS: u32 = 1_000; +const SDHCI_TUNING_POLLS: u32 = 1_000_000; +const SDHCI_VOLTAGE_SWITCH_DELAY_MS: u64 = 5; + /// Cloneable, sync-safe SDHCI IRQ top-half handle. #[derive(Clone)] pub struct SdhciIrqHandle { - base_addr: usize, - irq_state: *const host::IrqState, + irq: Arc, } -// SAFETY: The handle only performs volatile MMIO accesses and atomic cache -// updates. The owning `Sdhci` outlives handles created by OS glue. -unsafe impl Send for SdhciIrqHandle {} -// SAFETY: See the `Send` impl. -unsafe impl Sync for SdhciIrqHandle {} - -impl SdioHost for Sdhci { +impl ProtocolSdioHost for Sdhci { type Event = Event; type DataRequest<'a> = DataRequest<'a>; + type BusRequest = ReadyBusRequest; fn submit_command(&mut self, cmd: &Command) -> Result<(), Error> { + self.check_not_poisoned()?; Sdhci::submit_command(self, cmd) } @@ -206,35 +338,29 @@ impl SdioHost for Sdhci { } fn set_bus_width(&mut self, width: BusWidth) -> Result<(), Error> { - let mut ctrl = self.read_u8(REG_HOST_CONTROL1); - ctrl &= !(HOST_CTRL1_4BIT | HOST_CTRL1_8BIT); - match width { - BusWidth::Bit1 => {} - BusWidth::Bit4 => ctrl |= HOST_CTRL1_4BIT, - // 8-bit is eMMC territory and is intentionally not part of the - // MVP — surface it as Unsupported so the protocol layer can - // refuse cleanly instead of silently writing the bit and - // misconfiguring the bus. - BusWidth::Bit8 => return Err(Error::UnsupportedCommand), - // Future BusWidth variants are not supported by this controller. - _ => return Err(Error::UnsupportedCommand), - } - self.write_u8(REG_HOST_CONTROL1, ctrl); - Ok(()) + self.apply_bus_width(width) } fn set_clock(&mut self, speed: ClockSpeed) -> Result<(), Error> { - let target_hz = match speed { - ClockSpeed::Identification => 400_000, - ClockSpeed::Default | ClockSpeed::Sdr12 => 25_000_000, - ClockSpeed::HighSpeed | ClockSpeed::Sdr25 => 50_000_000, - ClockSpeed::Sdr50 | ClockSpeed::Ddr50 => 50_000_000, - ClockSpeed::Sdr104 => 104_000_000, - ClockSpeed::Hs200 => 200_000_000, + let (target_hz, uhs_mode) = match speed { + ClockSpeed::Identification => (400_000, HOST_CTRL2_UHS_SDR12), + ClockSpeed::Default | ClockSpeed::Sdr12 => (25_000_000, HOST_CTRL2_UHS_SDR12), + ClockSpeed::HighSpeed | ClockSpeed::Sdr25 => (50_000_000, HOST_CTRL2_UHS_SDR25), + ClockSpeed::Sdr50 => (50_000_000, HOST_CTRL2_UHS_SDR50), + ClockSpeed::Ddr50 => (50_000_000, HOST_CTRL2_UHS_DDR50), + ClockSpeed::Sdr104 => (104_000_000, HOST_CTRL2_UHS_SDR104), + ClockSpeed::Hs200 => (200_000_000, HOST_CTRL2_UHS_SDR104), // Future ClockSpeed variants are not supported by this controller. _ => return Err(Error::UnsupportedCommand), }; + // Match Linux's SDHCI/DWCMSHC UHS signaling selection: even legacy + // MMC HighSpeed maps to the SDR25 bus-speed mode on controllers that + // interpret HOST_CONTROL2.UHS_MODE_SELECT. + let mut ctrl2 = self.read_u16(REG_HOST_CONTROL2); + ctrl2 = (ctrl2 & !HOST_CTRL2_UHS_MODE_MASK) | uhs_mode; + self.write_u16(REG_HOST_CONTROL2, ctrl2); + // Toggle the High-Speed Enable bit in HOST_CONTROL1 alongside the // divider change so the controller pipelines reflect the new // timing window. @@ -282,6 +408,9 @@ impl SdioHost for Sdhci { if matches!(voltage, SignalVoltage::V180) && !self.support_1v8 { return Err(Error::UnsupportedCommand); } + if matches!(voltage, SignalVoltage::V120) { + return Err(Error::UnsupportedCommand); + } self.disable_sd_clock(); @@ -298,7 +427,7 @@ impl SdioHost for Sdhci { ctrl2 |= HOST_CTRL2_1V8_SIGNALING; self.set_power(POWER_180); } - SignalVoltage::V120 => return Err(Error::UnsupportedCommand), + SignalVoltage::V120 => unreachable!("V120 was rejected before mutating registers"), // Future SignalVoltage variants are not supported by this controller. _ => return Err(Error::UnsupportedCommand), } @@ -318,7 +447,11 @@ impl SdioHost for Sdhci { Ok(()) } - fn execute_tuning(&mut self, cmd_index: u8) -> Result<(), Error> { + fn execute_tuning( + &mut self, + cmd_index: u8, + block_size: core::num::NonZeroU16, + ) -> Result<(), Error> { // Only CMD19 (SD UHS-I) and CMD21 (eMMC HS200) make sense here. // Reject anything else loudly so the protocol layer doesn't // accidentally tune for a non-tuning command. @@ -327,20 +460,21 @@ impl SdioHost for Sdhci { } // Block size for the tuning data phase: SD CMD19 always 64, - // MMC CMD21 is 64 (4-bit) or 128 (8-bit). The host doesn't - // know the bus width here without snooping HOST_CONTROL1; we - // read it back to pick the right size. - let block_size: u16 = + // MMC CMD21 is 64 (4-bit) or 128 (8-bit). + let expected_block_size = if cmd_index == 21 && self.read_u8(REG_HOST_CONTROL1) & HOST_CTRL1_8BIT != 0 { - 128 + sdmmc_protocol::cmd::MMC_TUNING_BLOCK_SIZE_8BIT } else { - 64 + sdmmc_protocol::cmd::SD_TUNING_BLOCK_SIZE }; + if u32::from(block_size.get()) != expected_block_size { + return Err(Error::InvalidArgument); + } // Pre-program the data registers per SDHCI v3 §3.7.7. The // controller issues the tuning command itself; we just hand it // the shape of the data phase. - self.write_u16(REG_BLOCK_SIZE, block_size & 0x0FFF); + self.write_u16(REG_BLOCK_SIZE, block_size.get() & 0x0FFF); self.write_u16(REG_BLOCK_COUNT, 1); self.write_u8(REG_TIMEOUT_CONTROL, 0x0E); // Direction = read, single block, DMA disabled. @@ -402,6 +536,14 @@ impl SdioHost for Sdhci { fn handle_irq(&mut self) -> Self::Event { self.irq_handle().handle_irq() } + + fn submit_bus_op(&mut self, op: SdioBusOp) -> Result { + submit_ready_bus_op(self, op) + } + + fn poll_bus_op(&mut self, request: &mut Self::BusRequest) -> Result, Error> { + poll_ready_bus_op(request) + } } impl SdioIrqHost for Sdhci { @@ -416,6 +558,944 @@ impl SdioIrqHost for Sdhci { } } +impl sdio_host2::SdioHost for Sdhci { + type TransactionRequest<'a> + = TransactionRequest<'a> + where + Self: 'a; + type BusRequest = BusRequest; + + unsafe fn submit_transaction<'a>( + &mut self, + transaction: sdio_host2::Transaction<'a>, + ) -> Result, sdio_host2::Error> + where + Self: 'a, + { + self.check_not_poisoned().map_err(map_protocol_error)?; + if !self.physical_bus_idle() { + return Err(sdio_host2::Error::Busy); + } + let owner = self.host2_owner(); + let id = self.start_host2_request(); + let response = transaction.command.response; + match transaction.data { + None => { + if let Err(err) = self.submit_command(&transaction.command) { + self.finish_host2_request(id); + return Err(map_protocol_error(err)); + } + Ok(TransactionRequest::command(owner, id, response)) + } + Some(phase) => { + phase + .validate() + .inspect_err(|_| self.finish_host2_request(id))?; + let block_size = u32::from(phase.block_size.get()); + let block_count = phase.block_count.get(); + let request = match phase.buffer { + sdio_host2::DataBuffer::Read(buf) => { + if !matches!(phase.direction, sdio_host2::DataDirection::Read) { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + ::submit_read_data( + self, + &transaction.command, + buf, + block_size, + block_count, + ) + } + sdio_host2::DataBuffer::Write(buf) => { + if !matches!(phase.direction, sdio_host2::DataDirection::Write) { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + ::submit_write_data( + self, + &transaction.command, + buf, + block_size, + block_count, + ) + } + sdio_host2::DataBuffer::Dma(_) => { + self.finish_host2_request(id); + return Err(sdio_host2::Error::InvalidArgument); + } + } + .inspect_err(|_| self.finish_host2_request(id)) + .map_err(map_protocol_error)?; + Ok(TransactionRequest::data(owner, id, request, response)) + } + } + } + + unsafe fn submit_transaction_owned<'a>( + &mut self, + transaction: sdio_host2::Transaction<'a>, + ) -> Result, sdio_host2::SubmitTransactionError<'a>> + where + Self: 'a, + { + if let Err(err) = self.check_not_poisoned() { + return Err(sdio_host2::SubmitTransactionError::new( + map_protocol_error(err), + transaction, + )); + } + if !matches!( + transaction.data.as_ref().map(|data| &data.buffer), + Some(sdio_host2::DataBuffer::Dma(_)) + ) { + return unsafe { self.submit_transaction(transaction) } + .map_err(sdio_host2::SubmitTransactionError::consumed); + } + if !self.physical_bus_idle() { + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Busy, + transaction, + )); + } + + let owner = self.host2_owner(); + let host2_id = self.start_host2_request(); + let response = transaction.command.response; + let Some(phase) = transaction.data else { + unreachable!("DMA transaction must contain a data phase") + }; + let block_size = u32::from(phase.block_size.get()); + let block_count = phase.block_count.get(); + let sdio_host2::DataBuffer::Dma(buffer) = phase.buffer else { + unreachable!("checked for DMA data buffer above") + }; + if !should_try_dma( + &transaction.command, + block_size, + block_count, + buffer.len().get(), + match phase.direction { + sdio_host2::DataDirection::Read => DataDirection::Read, + sdio_host2::DataDirection::Write => DataDirection::Write, + _ => { + self.finish_host2_request(host2_id); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + sdio_host2::Transaction::with_data(transaction.command, data), + )); + } + }, + ) { + self.finish_host2_request(host2_id); + let tx = sdio_host2::Transaction::with_data( + transaction.command, + sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }, + ); + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + tx, + )); + } + let Some(dma) = self.dma.clone() else { + self.finish_host2_request(host2_id); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + return Err(sdio_host2::SubmitTransactionError::new( + sdio_host2::Error::Unsupported, + sdio_host2::Transaction::with_data(transaction.command, data), + )); + }; + let mut slot = BlockRequestSlot::default(); + let submit = match phase.direction { + sdio_host2::DataDirection::Read => self.submit_prepared_read_blocks( + transaction.command.argument, + buffer, + &dma, + &mut slot, + ), + sdio_host2::DataDirection::Write => self.submit_prepared_write_blocks( + transaction.command.argument, + buffer, + &dma, + &mut slot, + ), + _ => unreachable!("unsupported direction returned before submit"), + }; + match submit { + Ok(request) => { + let id = request.id(); + let data = DataRequest { + id, + request: Some(request), + slot, + _buffer: PhantomData, + }; + Ok(TransactionRequest::data(owner, host2_id, data, response)) + } + Err(err) => { + self.finish_host2_request(host2_id); + let error = err.error; + let buffer = err.into_buffer(); + let data = sdio_host2::DataPhase { + direction: phase.direction, + block_size: phase.block_size, + block_count: phase.block_count, + buffer: sdio_host2::DataBuffer::Dma(buffer), + }; + Err(sdio_host2::SubmitTransactionError::new( + map_protocol_error(error), + sdio_host2::Transaction::with_data(transaction.command, data), + )) + } + } + } + + fn poll_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result, sdio_host2::PollRequestError> + where + Self: 'a, + { + self.check_host2_transaction_request(request)?; + match request.kind { + TransactionRequestKind::Command { response } => { + match ::poll_command_response(self) { + Ok(sdmmc_protocol::CommandResponsePoll::Pending) => { + Ok(sdio_host2::RequestPoll::Pending) + } + Ok(sdmmc_protocol::CommandResponsePoll::Complete(resp)) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok( + resp.to_raw_response(response) + ))) + } + Ok(_) => Ok(sdio_host2::RequestPoll::Pending), + Err(err) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(map_protocol_error(err)))) + } + } + } + TransactionRequestKind::Data { response } => { + let Some(data) = request.data.as_mut() else { + let recovery = self.abort_host2_transaction_request(request).err(); + return Ok(sdio_host2::RequestPoll::Ready(Err( + recovery.unwrap_or(sdio_host2::Error::InvalidArgument) + ))); + }; + match ::poll_data_request(self, data) { + Ok(DataCommandPoll::Pending) => Ok(sdio_host2::RequestPoll::Pending), + Ok(DataCommandPoll::Complete(resp)) => { + self.complete_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok( + resp.to_raw_response(response) + ))) + } + Ok(_) => Ok(sdio_host2::RequestPoll::Pending), + Err(err) => { + let _ = self.abort_host2_transaction_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(map_protocol_error(err)))) + } + } + } + } + } + + fn abort_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result<(), sdio_host2::Error> + where + Self: 'a, + { + if request.done { + return Ok(()); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::Error::InvalidArgument); + } + self.abort_host2_transaction_request(request) + } + + fn take_completed_dma<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Option + where + Self: 'a, + { + request + .data + .as_mut() + .and_then(|data| data.slot.take_completed_dma()) + } + + unsafe fn submit_bus_op( + &mut self, + op: sdio_host2::BusOp, + ) -> Result { + self.check_not_poisoned().map_err(map_protocol_error)?; + if !self.physical_bus_idle() { + return Err(sdio_host2::Error::Busy); + } + let state = self.prepare_host2_bus_op(op)?; + let owner = self.host2_owner(); + let id = self.start_host2_request(); + Ok(BusRequest::pending(owner, id, state)) + } + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, sdio_host2::PollRequestError> { + self.check_host2_bus_request(request)?; + match self.poll_host2_bus_state(&mut request.state) { + Ok(sdio_host2::RequestPoll::Pending) => Ok(sdio_host2::RequestPoll::Pending), + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) => { + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + Ok(sdio_host2::RequestPoll::Ready(Err(err))) => { + let _ = self.abort_host2_bus_state(&mut request.state); + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(err))) + } + Err(err) => { + let _ = self.abort_host2_bus_state(&mut request.state); + self.complete_host2_bus_request(request); + Ok(sdio_host2::RequestPoll::Ready(Err(err))) + } + } + } + + fn abort_bus_op(&mut self, request: &mut Self::BusRequest) -> Result<(), sdio_host2::Error> { + if request.done { + return Ok(()); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::Error::InvalidArgument); + } + let result = self.abort_host2_bus_state(&mut request.state); + request.done = true; + self.finish_host2_request(request.id); + result + } + + fn now_ms(&self) -> Option { + self.timer.map(HostTimer::now_ms) + } +} + +impl Sdhci { + fn physical_bus_idle(&self) -> bool { + matches!(self.command_state, command::CommandState::Idle) + && self.pending_data.is_none() + && self.host2_active_id.is_none() + } + + fn start_host2_request(&mut self) -> u64 { + let id = self.host2_next_id; + self.host2_next_id = self.host2_next_id.wrapping_add(1); + self.host2_active_id = Some(id); + id + } + + fn host2_owner(&self) -> usize { + self.base_addr + } + + fn finish_host2_request(&mut self, id: u64) { + if self.host2_active_id == Some(id) { + self.host2_active_id = None; + } + } + + fn prepare_host2_bus_op( + &self, + op: sdio_host2::BusOp, + ) -> Result { + match op { + sdio_host2::BusOp::ResetAll => Ok(BusRequestState::Reset { + mask: RESET_ALL, + phase: Phase::Init, + was_irq_enabled: self.completion_irq_enabled(), + started: false, + polls: 0, + }), + sdio_host2::BusOp::ResetCommandLine => Ok(BusRequestState::Reset { + mask: RESET_CMD, + phase: Phase::CommandSend, + was_irq_enabled: self.completion_irq_enabled(), + started: false, + polls: 0, + }), + sdio_host2::BusOp::ResetDataLine => Ok(BusRequestState::Reset { + mask: RESET_DAT, + phase: Phase::DataRead, + was_irq_enabled: self.completion_irq_enabled(), + started: false, + polls: 0, + }), + sdio_host2::BusOp::PowerOn => Ok(BusRequestState::PowerOn), + sdio_host2::BusOp::PowerOff => Ok(BusRequestState::PowerOff), + sdio_host2::BusOp::SetClock(speed) => self.prepare_host2_clock(speed), + sdio_host2::BusOp::SetClockHz(sdio_host2::ClockHz(hz)) => { + if self.base_clock_hz() == 0 { + return Err(sdio_host2::Error::Controller); + } + Ok(BusRequestState::SetClock(SdhciClockState::Start { + target_hz: hz, + uhs_mode: None, + high_speed: None, + })) + } + sdio_host2::BusOp::SetBusWidth(width) => match width { + BusWidth::Bit1 | BusWidth::Bit4 | BusWidth::Bit8 => { + Ok(BusRequestState::SetBusWidth(width)) + } + _ => Err(sdio_host2::Error::Unsupported), + }, + sdio_host2::BusOp::SetSignalVoltage(voltage) => self.prepare_host2_voltage(voltage), + sdio_host2::BusOp::ExecuteTuning { + command, + block_size, + } => self.prepare_host2_tuning(command, block_size), + _ => Err(sdio_host2::Error::Unsupported), + } + } + + fn prepare_host2_clock(&self, speed: ClockSpeed) -> Result { + let (target_hz, uhs_mode) = match speed { + ClockSpeed::Identification => (400_000, HOST_CTRL2_UHS_SDR12), + ClockSpeed::Default | ClockSpeed::Sdr12 => (25_000_000, HOST_CTRL2_UHS_SDR12), + ClockSpeed::HighSpeed | ClockSpeed::Sdr25 => (50_000_000, HOST_CTRL2_UHS_SDR25), + ClockSpeed::Sdr50 => (50_000_000, HOST_CTRL2_UHS_SDR50), + ClockSpeed::Ddr50 => (50_000_000, HOST_CTRL2_UHS_DDR50), + ClockSpeed::Sdr104 => (104_000_000, HOST_CTRL2_UHS_SDR104), + ClockSpeed::Hs200 => (200_000_000, HOST_CTRL2_UHS_SDR104), + _ => return Err(sdio_host2::Error::Unsupported), + }; + if self.ext_clock.is_none() && self.base_clock_hz() == 0 { + return Err(sdio_host2::Error::Controller); + } + let high_speed = !matches!( + speed, + ClockSpeed::Identification | ClockSpeed::Default | ClockSpeed::Sdr12 + ); + Ok(BusRequestState::SetClock(SdhciClockState::Start { + target_hz, + uhs_mode: Some(uhs_mode), + high_speed: Some(high_speed), + })) + } + + fn prepare_host2_voltage( + &self, + voltage: SignalVoltage, + ) -> Result { + if matches!(voltage, SignalVoltage::V180) && !self.support_1v8 { + return Err(sdio_host2::Error::Unsupported); + } + if matches!(voltage, SignalVoltage::V180) && self.timer.is_none() { + return Err(sdio_host2::Error::Unsupported); + } + match voltage { + SignalVoltage::V330 | SignalVoltage::V180 => Ok(BusRequestState::SetSignalVoltage( + SdhciVoltageState::DisableClock(voltage), + )), + SignalVoltage::V120 => Err(sdio_host2::Error::Unsupported), + _ => Err(sdio_host2::Error::Unsupported), + } + } + + fn prepare_host2_tuning( + &self, + command: sdio_host2::Command, + block_size: core::num::NonZeroU16, + ) -> Result { + if command.index != 19 && command.index != 21 { + return Err(sdio_host2::Error::InvalidArgument); + } + let expected = + if command.index == 21 && self.read_u8(REG_HOST_CONTROL1) & HOST_CTRL1_8BIT != 0 { + sdmmc_protocol::cmd::MMC_TUNING_BLOCK_SIZE_8BIT + } else { + sdmmc_protocol::cmd::SD_TUNING_BLOCK_SIZE + }; + if u32::from(block_size.get()) != expected { + return Err(sdio_host2::Error::InvalidArgument); + } + Ok(BusRequestState::ExecuteTuning(SdhciTuningState::Start { + cmd_index: command.index, + block_size: block_size.get(), + })) + } + + fn poll_host2_bus_state( + &mut self, + state: &mut BusRequestState, + ) -> Result, sdio_host2::Error> { + match state { + BusRequestState::Reset { + mask, + phase, + was_irq_enabled, + started, + polls, + } => self.poll_host2_reset(*mask, *phase, *was_irq_enabled, started, polls), + BusRequestState::PowerOn => { + self.set_power(POWER_330); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::PowerOff => { + self.write_u8(REG_POWER_CONTROL, 0); + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::SetClock(clock) => self.poll_host2_clock(clock), + BusRequestState::SetBusWidth(width) => { + self.apply_bus_width(*width).map_err(map_protocol_error)?; + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + BusRequestState::SetSignalVoltage(voltage) => self.poll_host2_voltage(voltage), + BusRequestState::ExecuteTuning(tuning) => self.poll_host2_tuning(tuning), + } + } + + fn poll_host2_reset( + &mut self, + mask: u8, + phase: Phase, + was_irq_enabled: bool, + started: &mut bool, + polls: &mut u32, + ) -> Result, sdio_host2::Error> { + if !*started { + self.write_u8(REG_SOFTWARE_RESET, mask); + *started = true; + } + if self.read_u8(REG_SOFTWARE_RESET) & mask == 0 { + if mask == RESET_ALL { + if let Some(hook) = self.reset_hook { + hook.after_reset(self).map_err(map_protocol_error)?; + } + self.restore_completion_irq_after_reset(was_irq_enabled); + } + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + if *polls >= SDHCI_RESET_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new(phase)))); + } + *polls += 1; + Ok(sdio_host2::RequestPoll::Pending) + } + + fn poll_host2_clock( + &mut self, + state: &mut SdhciClockState, + ) -> Result, sdio_host2::Error> { + match *state { + SdhciClockState::Start { + target_hz, + uhs_mode, + high_speed, + } => { + if let Some(mode) = uhs_mode { + let ctrl2 = + (self.read_u16(REG_HOST_CONTROL2) & !HOST_CTRL2_UHS_MODE_MASK) | mode; + self.write_u16(REG_HOST_CONTROL2, ctrl2); + } + if let Some(enabled) = high_speed { + let mut ctrl = self.read_u8(REG_HOST_CONTROL1); + if enabled { + ctrl |= HOST_CTRL1_HIGH_SPEED; + } else { + ctrl &= !HOST_CTRL1_HIGH_SPEED; + } + self.write_u8(REG_HOST_CONTROL1, ctrl); + } + if self.ext_clock.is_some() { + self.disable_sd_clock(); + *state = SdhciClockState::ExternalSetClock { target_hz }; + } else { + self.start_internal_clock(target_hz)?; + *state = SdhciClockState::InternalWaitStable { polls: 0 }; + } + Ok(sdio_host2::RequestPoll::Pending) + } + SdhciClockState::ExternalSetClock { target_hz } => { + let clock = self.ext_clock.ok_or(sdio_host2::Error::Controller)?; + clock.set_clock(target_hz).map_err(map_protocol_error)?; + self.start_external_clock(); + *state = SdhciClockState::ExternalEnable { polls: 0 }; + Ok(sdio_host2::RequestPoll::Pending) + } + SdhciClockState::ExternalEnable { ref mut polls } + | SdhciClockState::InternalWaitStable { ref mut polls } => { + self.poll_clock_stable(polls) + } + } + } + + fn start_internal_clock(&mut self, target_hz: u32) -> Result<(), sdio_host2::Error> { + self.write_u16(REG_CLOCK_CONTROL, 0); + if target_hz == 0 { + return Ok(()); + } + let base_clock_hz = self.base_clock_hz(); + if base_clock_hz == 0 { + return Err(sdio_host2::Error::Controller); + } + let div = sdhci_clock_divisor(base_clock_hz, target_hz); + let clk_ctrl = ((div & 0xFF) << 8) | ((div & 0x300) >> 2) | CLOCK_INTERNAL_ENABLE; + self.write_u16(REG_CLOCK_CONTROL, clk_ctrl); + Ok(()) + } + + fn start_external_clock(&mut self) { + self.write_u16(REG_CLOCK_CONTROL, 0); + self.write_u16(REG_CLOCK_CONTROL, CLOCK_INTERNAL_ENABLE); + } + + fn poll_clock_stable( + &mut self, + polls: &mut u32, + ) -> Result, sdio_host2::Error> { + let clock = self.read_u16(REG_CLOCK_CONTROL); + if clock & CLOCK_INTERNAL_ENABLE == 0 { + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + if clock & CLOCK_INTERNAL_STABLE != 0 { + self.write_u16(REG_CLOCK_CONTROL, clock | CLOCK_SD_ENABLE); + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + if *polls >= SDHCI_CLOCK_POLLS { + return Err(map_protocol_error(Error::Timeout(ErrorContext::new( + Phase::Init, + )))); + } + *polls += 1; + Ok(sdio_host2::RequestPoll::Pending) + } + + fn poll_host2_voltage( + &mut self, + state: &mut SdhciVoltageState, + ) -> Result, sdio_host2::Error> { + match *state { + SdhciVoltageState::DisableClock(voltage) => { + self.disable_sd_clock(); + *state = SdhciVoltageState::SwitchControllerAndRail(voltage); + Ok(sdio_host2::RequestPoll::Pending) + } + SdhciVoltageState::SwitchControllerAndRail(voltage) => { + if matches!(voltage, SignalVoltage::V180) && !self.dat_3_0_lines_low() { + self.rollback_host2_voltage(); + return Ok(sdio_host2::RequestPoll::Ready(Err( + sdio_host2::Error::Controller, + ))); + } + let mut ctrl2 = self.read_u16(REG_HOST_CONTROL2); + match voltage { + SignalVoltage::V330 => { + ctrl2 &= !HOST_CTRL2_1V8_SIGNALING; + self.set_power(POWER_330); + } + SignalVoltage::V180 => { + ctrl2 |= HOST_CTRL2_1V8_SIGNALING; + self.set_power(POWER_180); + } + SignalVoltage::V120 => return Err(sdio_host2::Error::Unsupported), + _ => return Err(sdio_host2::Error::Unsupported), + } + self.write_u16(REG_HOST_CONTROL2, ctrl2); + *state = SdhciVoltageState::WaitVsw { + voltage, + deadline_ms: self + .now_ms() + .map(|now| now.saturating_add(SDHCI_VOLTAGE_SWITCH_DELAY_MS)), + }; + Ok(sdio_host2::RequestPoll::Pending) + } + SdhciVoltageState::WaitVsw { + voltage, + deadline_ms, + } => { + if deadline_ms.is_none() + || deadline_ms + .zip(self.now_ms()) + .is_some_and(|(deadline, now)| now >= deadline) + { + *state = SdhciVoltageState::EnableClock(voltage); + } + Ok(sdio_host2::RequestPoll::Pending) + } + SdhciVoltageState::EnableClock(voltage) => { + let cur = self.read_u16(REG_CLOCK_CONTROL); + self.write_u16(REG_CLOCK_CONTROL, cur | CLOCK_SD_ENABLE); + *state = SdhciVoltageState::VerifyDatLines(voltage); + Ok(sdio_host2::RequestPoll::Pending) + } + SdhciVoltageState::VerifyDatLines(voltage) => { + if matches!(voltage, SignalVoltage::V180) && !self.dat_3_0_lines_high() { + self.rollback_host2_voltage(); + return Ok(sdio_host2::RequestPoll::Ready(Err( + sdio_host2::Error::Controller, + ))); + } + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + } + } + + fn poll_host2_tuning( + &mut self, + state: &mut SdhciTuningState, + ) -> Result, sdio_host2::Error> { + match *state { + SdhciTuningState::Start { + cmd_index, + block_size, + } => { + self.write_u16(REG_BLOCK_SIZE, block_size & 0x0FFF); + self.write_u16(REG_BLOCK_COUNT, 1); + self.write_u8(REG_TIMEOUT_CONTROL, 0x0E); + self.write_u16( + REG_TRANSFER_MODE, + XFER_MODE_BLOCK_COUNT_ENABLE | XFER_MODE_READ, + ); + let ctrl2 = self.read_u16(REG_HOST_CONTROL2) | HOST_CTRL2_EXECUTE_TUNING; + self.write_u16(REG_HOST_CONTROL2, ctrl2); + *state = SdhciTuningState::Wait { + cmd_index, + polls: 0, + }; + Ok(sdio_host2::RequestPoll::Pending) + } + SdhciTuningState::Wait { + cmd_index, + ref mut polls, + } => { + let status = self.read_u16(REG_HOST_CONTROL2); + if status & HOST_CTRL2_EXECUTE_TUNING == 0 { + if status & HOST_CTRL2_SAMPLING_CLOCK_SELECT != 0 { + return Ok(sdio_host2::RequestPoll::Ready(Ok(()))); + } + return Err(map_protocol_error(Error::BadResponse( + ErrorContext::for_cmd(Phase::Init, cmd_index), + ))); + } + if *polls >= SDHCI_TUNING_POLLS { + self.write_u16(REG_HOST_CONTROL2, status & !HOST_CTRL2_EXECUTE_TUNING); + return Err(map_protocol_error(Error::Timeout(ErrorContext::for_cmd( + Phase::Init, + cmd_index, + )))); + } + *polls += 1; + Ok(sdio_host2::RequestPoll::Pending) + } + } + } + + fn abort_host2_bus_state( + &mut self, + state: &mut BusRequestState, + ) -> Result<(), sdio_host2::Error> { + match state { + BusRequestState::Reset { mask, started, .. } if *started => { + if !self.reset_with_mask_best_effort(*mask) { + return Err(sdio_host2::Error::Timeout); + } + } + BusRequestState::SetClock(_) => self.reset_controller_for_host2_abort()?, + BusRequestState::SetSignalVoltage(_) => self.rollback_host2_voltage(), + BusRequestState::ExecuteTuning(SdhciTuningState::Wait { .. }) => { + let ctrl2 = self.read_u16(REG_HOST_CONTROL2) & !HOST_CTRL2_EXECUTE_TUNING; + self.write_u16(REG_HOST_CONTROL2, ctrl2); + self.reset_controller_for_host2_abort()?; + } + _ => {} + } + Ok(()) + } + + fn reset_controller_for_host2_abort(&mut self) -> Result<(), sdio_host2::Error> { + let was_irq_enabled = self.completion_irq_enabled(); + self.write_u8(REG_SOFTWARE_RESET, RESET_ALL); + if !self.reset_with_mask_best_effort(RESET_ALL) { + return Err(sdio_host2::Error::Timeout); + } + if let Some(hook) = self.reset_hook { + hook.after_reset(self).map_err(map_protocol_error)?; + } + self.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_CLEAR_ALL); + self.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_CLEAR_ALL); + self.clear_cached_irq_status(); + self.restore_completion_irq_after_reset(was_irq_enabled); + self.pending_data = None; + self.command_state = command::CommandState::Idle; + Ok(()) + } + + fn restore_completion_irq_after_reset(&mut self, was_irq_enabled: bool) { + self.enable_interrupts(); + if was_irq_enabled { + self.enable_completion_irq(); + } + } + + fn rollback_host2_voltage(&mut self) { + self.disable_sd_clock(); + let ctrl2 = self.read_u16(REG_HOST_CONTROL2) & !HOST_CTRL2_1V8_SIGNALING; + self.write_u16(REG_HOST_CONTROL2, ctrl2); + self.set_power(POWER_330); + let clock = self.read_u16(REG_CLOCK_CONTROL); + self.write_u16(REG_CLOCK_CONTROL, clock | CLOCK_SD_ENABLE); + } + + fn dat_3_0_lines_high(&self) -> bool { + self.read_u32(REG_PRESENT_STATE) & PRESENT_DAT_3_0_LINE_SIGNAL_LEVEL + == PRESENT_DAT_3_0_LINE_SIGNAL_LEVEL + } + + fn dat_3_0_lines_low(&self) -> bool { + self.read_u32(REG_PRESENT_STATE) & PRESENT_DAT_3_0_LINE_SIGNAL_LEVEL == 0 + } + + fn reset_with_mask_best_effort(&mut self, mask: u8) -> bool { + for _ in 0..SDHCI_RESET_POLLS { + if self.read_u8(REG_SOFTWARE_RESET) & mask == 0 { + return true; + } + core::hint::spin_loop(); + } + false + } + + fn apply_bus_width(&mut self, width: BusWidth) -> Result<(), Error> { + let mut ctrl = self.read_u8(REG_HOST_CONTROL1); + ctrl &= !(HOST_CTRL1_4BIT | HOST_CTRL1_8BIT); + match width { + BusWidth::Bit1 => {} + BusWidth::Bit4 => ctrl |= HOST_CTRL1_4BIT, + BusWidth::Bit8 => ctrl |= HOST_CTRL1_8BIT, + _ => return Err(Error::UnsupportedCommand), + } + self.write_u8(REG_HOST_CONTROL1, ctrl); + Ok(()) + } + + fn check_host2_transaction_request( + &self, + request: &TransactionRequest<'_>, + ) -> Result<(), sdio_host2::PollRequestError> { + if request.done { + return Err(sdio_host2::PollRequestError::AlreadyCompleted); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::PollRequestError::WrongOwner); + } + if self.host2_active_id != Some(request.id) { + return Err(sdio_host2::PollRequestError::StaleGeneration); + } + Ok(()) + } + + fn check_host2_bus_request( + &self, + request: &BusRequest, + ) -> Result<(), sdio_host2::PollRequestError> { + if request.done { + return Err(sdio_host2::PollRequestError::AlreadyCompleted); + } + if request.owner != self.host2_owner() { + return Err(sdio_host2::PollRequestError::WrongOwner); + } + if self.host2_active_id != Some(request.id) { + return Err(sdio_host2::PollRequestError::StaleGeneration); + } + Ok(()) + } + + fn complete_host2_transaction_request(&mut self, request: &mut TransactionRequest<'_>) { + request.done = true; + self.finish_host2_request(request.id); + } + + fn complete_host2_bus_request(&mut self, request: &mut BusRequest) { + request.done = true; + self.finish_host2_request(request.id); + } + + fn abort_host2_transaction_request( + &mut self, + request: &mut TransactionRequest<'_>, + ) -> Result<(), sdio_host2::Error> { + let result = if let Some(data) = request.data.as_mut() { + if let Some(active) = data.request.take() { + let id = active.id(); + let mut pending = Some(active); + self.abort_block_request_response(&mut pending, id, &mut data.slot) + .map_err(map_protocol_error) + } else { + Ok(()) + } + } else { + self.abort_command().map_err(map_protocol_error) + }; + request.done = true; + self.finish_host2_request(request.id); + result + } +} + +fn map_protocol_error(err: Error) -> sdio_host2::Error { + match err { + Error::Timeout(_) => sdio_host2::Error::Timeout, + Error::Crc(_) => sdio_host2::Error::Crc, + Error::NoCard => sdio_host2::Error::NoCard, + Error::Busy => sdio_host2::Error::Busy, + Error::UnsupportedCommand => sdio_host2::Error::Unsupported, + Error::Misaligned => sdio_host2::Error::Misaligned, + Error::InvalidArgument => sdio_host2::Error::InvalidArgument, + Error::BusError(_) => sdio_host2::Error::Bus, + Error::ReadError(_) | Error::WriteError(_) | Error::BadResponse(_) => { + sdio_host2::Error::Bus + } + Error::CardError(_) | Error::CardLocked => sdio_host2::Error::Controller, + _ => sdio_host2::Error::Controller, + } +} + +fn sdhci_clock_divisor(base_clock_hz: u32, target_hz: u32) -> u16 { + if target_hz == 0 || base_clock_hz <= target_hz { + return 0; + } + for n in 1..=0x3FF { + if base_clock_hz / (2 * n as u32) <= target_hz { + return n; + } + } + 0x3FF +} + fn submit_read_with_dma_fifo_fallback( host: &mut Sdhci, cmd: &Command, @@ -429,15 +1509,20 @@ fn submit_read_with_dma_fifo_fallback( && let Some(dma) = host.dma.clone() { match host.submit_read_blocks( - cmd.arg, + cmd.argument, buffer, NonZeroUsize::new(len).ok_or(Error::InvalidArgument)?, Some(&dma), BlockTransferMode::Dma, slot, ) { - Ok(request) => return Ok(request), - Err(err) if can_fallback_to_fifo(err) => {} + Ok(request) => { + log_adma_path_once("read"); + return Ok(request); + } + Err(err) if can_fallback_to_fifo(err) => { + log_adma_fallback_once("read", err); + } Err(err) => return Err(err), } } @@ -466,15 +1551,20 @@ fn submit_write_with_dma_fifo_fallback( && let Some(dma) = host.dma.clone() { match host.submit_write_blocks( - cmd.arg, + cmd.argument, buffer, NonZeroUsize::new(len).ok_or(Error::InvalidArgument)?, Some(&dma), BlockTransferMode::Dma, slot, ) { - Ok(request) => return Ok(request), - Err(err) if can_fallback_to_fifo(err) => {} + Ok(request) => { + log_adma_path_once("write"); + return Ok(request); + } + Err(err) if can_fallback_to_fifo(err) => { + log_adma_fallback_once("write", err); + } Err(err) => return Err(err), } } @@ -500,7 +1590,7 @@ fn should_try_dma( block_size == 512 && len == block_count as usize * 512 && matches!( - (direction, cmd.cmd), + (direction, cmd.index), (DataDirection::Read, 17 | 18) | (DataDirection::Write, 24 | 25) ) } @@ -512,13 +1602,39 @@ fn can_fallback_to_fifo(err: Error) -> bool { ) } +fn log_adma_path_once(direction: &str) { + let logged = match direction { + "read" => &ADMA_READ_PATH_LOGGED, + "write" => &ADMA_WRITE_PATH_LOGGED, + _ => return, + }; + if !logged.swap(true, Ordering::Relaxed) { + log::info!("sdhci: using ADMA2 {direction} data path"); + } +} + +fn log_adma_fallback_once(direction: &str, err: Error) { + let logged = match direction { + "read" => &ADMA_READ_FALLBACK_LOGGED, + "write" => &ADMA_WRITE_FALLBACK_LOGGED, + _ => return, + }; + if !logged.swap(true, Ordering::Relaxed) { + log::warn!("sdhci: falling back to FIFO for {direction} data path: {err:?}"); + } +} + pub(crate) fn event_from_status(normal: u16, error: u16) -> Event { if normal & NORMAL_INT_ERROR != 0 { Event::Error { normal, error } - } else if normal & NORMAL_INT_CMD_COMPLETE != 0 { - Event::CommandComplete } else if normal & NORMAL_INT_XFER_COMPLETE != 0 { Event::TransferComplete + } else if normal & NORMAL_INT_BUFFER_READ_READY != 0 { + Event::ReceiveReady + } else if normal & NORMAL_INT_BUFFER_WRITE_READY != 0 { + Event::TransmitReady + } else if normal & NORMAL_INT_CMD_COMPLETE != 0 { + Event::CommandComplete } else if normal != 0 || error != 0 { Event::Other { normal, error } } else { @@ -532,6 +1648,8 @@ impl HostEvent for Event { Event::None => HostEventKind::None, Event::CommandComplete => HostEventKind::CommandComplete, Event::TransferComplete => HostEventKind::TransferComplete, + Event::ReceiveReady => HostEventKind::ReceiveReady, + Event::TransmitReady => HostEventKind::TransmitReady, Event::Error { .. } => HostEventKind::Error, Event::Other { .. } => HostEventKind::Other, } @@ -540,14 +1658,18 @@ impl HostEvent for Event { fn source(&self) -> HostEventSource { match self { Event::CommandComplete => HostEventSource::Command, - Event::TransferComplete => HostEventSource::Data, + Event::TransferComplete | Event::ReceiveReady | Event::TransmitReady => { + HostEventSource::Data + } Event::None | Event::Error { .. } | Event::Other { .. } => HostEventSource::Controller, } } fn queue_id(&self) -> Option { match self { - Event::TransferComplete => Some(BlockRequestId::new(0)), + Event::TransferComplete | Event::ReceiveReady | Event::TransmitReady => { + Some(BlockRequestId::new(0)) + } Event::None | Event::CommandComplete | Event::Error { .. } | Event::Other { .. } => { None } @@ -571,8 +1693,7 @@ impl Sdhci { pub fn irq_handle(&self) -> SdhciIrqHandle { SdhciIrqHandle { - base_addr: self.base_addr, - irq_state: &self.irq_state, + irq: self.irq.clone(), } } @@ -587,20 +1708,21 @@ impl SdioIrqHandle for SdhciIrqHandle { type Event = Event; fn handle_irq(&self) -> Self::Event { - let normal = read_u16(self.base_addr, REG_NORMAL_INT_STATUS); + let generation = self.irq.state.generation(); + let normal = read_u16(self.irq.base_addr, REG_NORMAL_INT_STATUS); let error = if normal & NORMAL_INT_ERROR != 0 { - read_u16(self.base_addr, REG_ERROR_INT_STATUS) + read_u16(self.irq.base_addr, REG_ERROR_INT_STATUS) } else { 0 }; if normal != 0 { - write_u16(self.base_addr, REG_NORMAL_INT_STATUS, normal); + write_u16(self.irq.base_addr, REG_NORMAL_INT_STATUS, normal); } if error != 0 { - write_u16(self.base_addr, REG_ERROR_INT_STATUS, error); + write_u16(self.irq.base_addr, REG_ERROR_INT_STATUS, error); } - unsafe { &*self.irq_state }.cache(normal, error); + self.irq.state.cache_if_current(generation, normal, error); event_from_status(normal, error) } @@ -616,6 +1738,10 @@ fn write_u16(base_addr: usize, off: usize, val: u16) { #[cfg(test)] mod tests { + use core::num::{NonZeroU16, NonZeroU32}; + + use sdio_host2::ResponseType; + use super::*; #[test] @@ -656,6 +1782,17 @@ mod tests { assert_eq!(event.queue_id(), Some(BlockRequestId::new(0))); } + #[test] + fn merged_command_and_data_irq_reports_queue_ready() { + use sdmmc_protocol::sdio::{HostEvent, HostEventKind, HostEventSource}; + + let event = event_from_status(NORMAL_INT_CMD_COMPLETE | NORMAL_INT_XFER_COMPLETE, 0); + + assert_eq!(event.kind(), HostEventKind::TransferComplete); + assert_eq!(event.source(), HostEventSource::Data); + assert_eq!(event.queue_id(), Some(BlockRequestId::new(0))); + } + #[test] fn exposes_block_buffer_constraints() { let host = unsafe { Sdhci::new_from_addr(0x1000_0000) }; @@ -666,6 +1803,135 @@ mod tests { assert_eq!(dma.dma_mask, Some(u32::MAX as u64)); } + #[test] + fn host2_data_submit_reports_busy_without_dirtying_pending_data() { + let mut host = unsafe { Sdhci::new_from_addr(0x1000_0000) }; + host.command_state = command::CommandState::Issued { + cmd: Command::new(0, 0, ResponseType::None), + data_line: false, + polls: 0, + }; + let mut buf = [0u8; 512]; + let data = sdio_host2::DataPhase::read( + NonZeroU16::new(512).unwrap(), + NonZeroU32::new(1).unwrap(), + &mut buf, + ) + .unwrap(); + let tx = sdio_host2::Transaction::with_data(Command::new(17, 0, ResponseType::R1), data); + + let err = + match unsafe { ::submit_transaction(&mut host, tx) } { + Ok(_) => panic!("busy host accepted a second transaction"), + Err(err) => err, + }; + + assert_eq!(err, sdio_host2::Error::Busy); + assert!(host.pending_data.is_none()); + } + + #[test] + fn host2_poll_after_complete_is_rejected() { + #[repr(align(4))] + struct FakeRegs([u8; 0x100]); + + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + let mut request = unsafe { + ::submit_bus_op(&mut host, sdio_host2::BusOp::PowerOn) + } + .unwrap(); + + assert!(matches!( + ::poll_bus_op(&mut host, &mut request), + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + )); + assert_eq!( + ::poll_bus_op(&mut host, &mut request), + Err(sdio_host2::PollRequestError::AlreadyCompleted) + ); + } + + #[test] + fn host2_bus_request_is_bound_to_originating_host() { + #[repr(align(4))] + struct FakeRegs([u8; 0x100]); + + let mut regs_a = FakeRegs([0; 0x100]); + let mut regs_b = FakeRegs([0; 0x100]); + let base_a = NonNull::new(regs_a.0.as_mut_ptr()).unwrap(); + let base_b = NonNull::new(regs_b.0.as_mut_ptr()).unwrap(); + let mut host_a = unsafe { Sdhci::new(base_a) }; + let mut host_b = unsafe { Sdhci::new(base_b) }; + let mut request = unsafe { + ::submit_bus_op(&mut host_a, sdio_host2::BusOp::PowerOn) + } + .unwrap(); + + assert_eq!( + ::poll_bus_op(&mut host_b, &mut request), + Err(sdio_host2::PollRequestError::WrongOwner) + ); + } + + #[test] + fn host2_v180_requires_real_timer() { + let mut host = unsafe { Sdhci::new_from_addr(0x1000_0000) }; + host.enable_1v8_signaling(); + + assert!(matches!( + unsafe { + ::submit_bus_op( + &mut host, + sdio_host2::BusOp::SetSignalVoltage(sdio_host2::SignalVoltage::V180), + ) + }, + Err(sdio_host2::Error::Unsupported) + )); + } + + #[test] + fn host2_v180_rejects_partial_high_dat_lines_before_switch() { + #[repr(align(4))] + struct FakeRegs([u8; 0x100]); + + struct StaticTimer; + + impl HostTimer for StaticTimer { + fn now_ms(&self) -> u64 { + 0 + } + } + + static TIMER: StaticTimer = StaticTimer; + + let mut regs = FakeRegs([0; 0x100]); + let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); + let mut host = unsafe { Sdhci::new(base) }; + host.enable_1v8_signaling(); + host.set_timer(&TIMER); + host.write_u32(REG_PRESENT_STATE, 1 << 20); + let mut request = unsafe { + ::submit_bus_op( + &mut host, + sdio_host2::BusOp::SetSignalVoltage(sdio_host2::SignalVoltage::V180), + ) + } + .unwrap(); + + assert!(matches!( + ::poll_bus_op(&mut host, &mut request), + Ok(sdio_host2::RequestPoll::Pending) + )); + assert!(matches!( + ::poll_bus_op(&mut host, &mut request), + Ok(sdio_host2::RequestPoll::Ready(Err( + sdio_host2::Error::Controller + ))) + )); + } + #[test] fn irq_handle_acks_and_caches_status_without_mutable_host() { #[repr(align(4))] @@ -674,6 +1940,7 @@ mod tests { let mut regs = FakeRegs([0; 0x100]); let base = NonNull::new(regs.0.as_mut_ptr()).unwrap(); let host = unsafe { Sdhci::new(base) }; + host.irq.state.begin_request(); host.write_u16(REG_NORMAL_INT_STATUS, NORMAL_INT_ERROR); host.write_u16(REG_ERROR_INT_STATUS, ERROR_INT_DATA_TIMEOUT); @@ -686,8 +1953,8 @@ mod tests { error: ERROR_INT_DATA_TIMEOUT, } ); - assert_eq!(host.irq_state.pending_normal(), NORMAL_INT_ERROR); - assert_eq!(host.irq_state.pending_error(), ERROR_INT_DATA_TIMEOUT); + assert_eq!(host.irq.state.pending_normal(), NORMAL_INT_ERROR); + assert_eq!(host.irq.state.pending_error(), ERROR_INT_DATA_TIMEOUT); host.write_u16(REG_NORMAL_INT_STATUS, 0); host.write_u16(REG_ERROR_INT_STATUS, 0); assert_eq!(host.handle_irq(), Event::None); diff --git a/drivers/blk/sdhci-host/src/rdif.rs b/drivers/blk/sdhci-host/src/rdif.rs new file mode 100644 index 0000000000..dded7edee9 --- /dev/null +++ b/drivers/blk/sdhci-host/src/rdif.rs @@ -0,0 +1,112 @@ +//! RDIF block-device adapter for [`Sdhci`]. + +use dma_api::DeviceDma; +pub use protocol_rdif::{BlockConfig, BlockDevice, BlockQueue}; +pub use rdif_block::{ + BInterface, BIrqHandler, BOwnedQueue, BQueue, BlkError, IQueue, IQueueOwned, Interface, + IrqHandlerHandle, IrqHandlerSlot, OwnedRequest, PollError, QueueHandle, Request, + RequestId as RdifRequestId, RequestPoll as OwnedRequestPoll, RequestStatus, SubmitError, +}; +use sdmmc_protocol::{ + rdif as protocol_rdif, + sdio::{SdioHost2Adapter, SdioSdmmc}, +}; + +use crate::{ADMA2_MAX_BLOCKS, ADMA2_MAX_TRANSFER_SIZE, Sdhci}; + +pub fn device( + card: SdioSdmmc>, + config: BlockConfig, +) -> BlockDevice> { + BlockDevice::new(card, config) +} + +pub fn dma_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, + dma: DeviceDma, +) -> BlockConfig { + BlockConfig::dma(name, capacity_blocks, irq_driven, dma) + .with_max_blocks_per_request(ADMA2_MAX_BLOCKS) + .with_max_segment_size(ADMA2_MAX_TRANSFER_SIZE) +} + +pub const fn fifo_config( + name: &'static str, + capacity_blocks: u64, + irq_driven: bool, +) -> BlockConfig { + BlockConfig::fifo(name, capacity_blocks, irq_driven) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fifo_config_keeps_one_block_limits() { + let config = fifo_config("sdhci", 16, true); + let limits = protocol_rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, 1); + assert_eq!(limits.max_segment_size, protocol_rdif::BLOCK_SIZE); + assert!(!config.uses_dma()); + } + + #[test] + fn dma_config_advertises_adma_window() { + let config = dma_config( + "sdhci", + 16, + true, + dma_api::DeviceDma::new_legacy(u32::MAX as u64, &TEST_DMA), + ); + let limits = protocol_rdif::queue_limits(&config, config.dma_mask); + + assert_eq!(limits.max_blocks_per_request, ADMA2_MAX_BLOCKS); + assert_eq!(limits.max_segment_size, ADMA2_MAX_TRANSFER_SIZE); + assert!(config.uses_dma()); + } + + struct TestDma; + static TEST_DMA: TestDma = TestDma; + + impl dma_api::DmaOp for TestDma { + fn page_size(&self) -> usize { + protocol_rdif::BLOCK_SIZE + } + + unsafe fn alloc_contiguous( + &self, + _constraints: dma_api::DmaConstraints, + _layout: core::alloc::Layout, + ) -> Option { + None + } + + unsafe fn dealloc_contiguous(&self, _handle: dma_api::DmaAllocHandle) {} + + unsafe fn alloc_coherent( + &self, + _constraints: dma_api::DmaConstraints, + _layout: core::alloc::Layout, + ) -> Option { + None + } + + unsafe fn dealloc_coherent(&self, _handle: dma_api::DmaAllocHandle) {} + + unsafe fn map_streaming( + &self, + _constraints: dma_api::DmaConstraints, + _addr: core::ptr::NonNull, + _size: core::num::NonZeroUsize, + _direction: dma_api::DmaDirection, + ) -> Result { + Err(dma_api::DmaError::NoMemory) + } + + unsafe fn unmap_streaming(&self, _handle: dma_api::DmaMapHandle) {} + } +} diff --git a/drivers/blk/sdhci-host/src/regs.rs b/drivers/blk/sdhci-host/src/regs.rs index 88899630a3..44061bce10 100644 --- a/drivers/blk/sdhci-host/src/regs.rs +++ b/drivers/blk/sdhci-host/src/regs.rs @@ -46,6 +46,8 @@ pub(crate) const PRESENT_DAT_INHIBIT: u32 = 1 << 1; pub(crate) const PRESENT_BUFFER_WRITE_ENABLE: u32 = 1 << 10; pub(crate) const PRESENT_BUFFER_READ_ENABLE: u32 = 1 << 11; pub(crate) const PRESENT_CARD_INSERTED: u32 = 1 << 16; +pub(crate) const PRESENT_DAT0_LINE_SIGNAL_LEVEL: u32 = 1 << 20; +pub(crate) const PRESENT_DAT_3_0_LINE_SIGNAL_LEVEL: u32 = 0x0F << 20; // ── Software Reset ───────────────────────────────────────────────────── diff --git a/drivers/blk/sdmmc-protocol/Cargo.toml b/drivers/blk/sdmmc-protocol/Cargo.toml index df58eb5c7a..fc37922f14 100644 --- a/drivers/blk/sdmmc-protocol/Cargo.toml +++ b/drivers/blk/sdmmc-protocol/Cargo.toml @@ -12,9 +12,13 @@ categories = ["embedded", "no-std", "hardware-support"] [dependencies] bitflags = "2" embedded-hal = "1" +dma-api.workspace = true log.workspace = true +rdif-block = { workspace = true, optional = true } +sdio-host2.workspace = true [features] default = ["spi"] spi = [] sdio = [] +rdif = ["sdio", "dep:rdif-block"] diff --git a/drivers/blk/sdmmc-protocol/src/cmd.rs b/drivers/blk/sdmmc-protocol/src/cmd.rs index 2eea15399e..bb1da87435 100644 --- a/drivers/blk/sdmmc-protocol/src/cmd.rs +++ b/drivers/blk/sdmmc-protocol/src/cmd.rs @@ -1,3 +1,5 @@ +pub use sdio_host2::Command; + use crate::response::ResponseType; /// Direction of the data phase that follows a command, if any. @@ -22,105 +24,6 @@ impl DataDirection { } } -/// SD/MMC command definitions -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Command { - pub cmd: u8, - pub arg: u32, - pub resp_type: ResponseType, -} - -impl Command { - pub const fn new(cmd: u8, arg: u32, resp_type: ResponseType) -> Self { - Self { - cmd, - arg, - resp_type, - } - } - - /// Return a copy of this command with `resp_type` overridden. - /// - /// Useful when the same command index has different response types depending - /// on the transport (e.g. ACMD41 returns R3 in native mode but the OCR is - /// not available in SPI mode where only an R1 byte is returned). - pub const fn with_resp_type(self, resp_type: ResponseType) -> Self { - Self { resp_type, ..self } - } - - /// Command index (0–63) - pub fn index(&self) -> u8 { - self.cmd - } - - /// 32-bit argument - pub fn argument(&self) -> u32 { - self.arg - } - - /// Direction of the data phase that follows this command. - /// - /// Note: SDIO CMD53 carries its direction in the argument; this helper - /// returns `None` for it. CMD6 is also returned as `None` because the - /// same command index is reused for ACMD6 (SET_BUS_WIDTH, no data phase) - /// and CMD6 SWITCH_FUNC (64-byte read). Drivers that issue SWITCH_FUNC - /// choose the read-data submit path explicitly. - pub const fn data_direction(&self) -> DataDirection { - match self.cmd { - 17 | 18 => DataDirection::Read, - 24 | 25 => DataDirection::Write, - _ => DataDirection::None, - } - } - - /// Size (in bytes) of the data block this command transfers, when the - /// answer is unambiguous from the command index alone. - /// - /// Returns `None` for commands without a data phase, for commands whose - /// block size depends on host configuration (e.g. CMD16-controlled - /// SDSC blocks), and for indices that are reused across commands with - /// different data shapes (e.g. CMD6). - pub const fn data_block_size(&self) -> Option { - match self.cmd { - 17 | 18 | 24 | 25 => Some(512), - _ => None, - } - } - - /// Compute the 7-bit CRC for SPI mode transmission - pub fn crc7(&self) -> u8 { - let mut crc: u8 = 0; - // The token is: 01 | cmd[5:0] - let token: u8 = 0x40 | (self.cmd & 0x3F); - crc = crc7_update(crc, token); - for byte in self.arg.to_be_bytes() { - crc = crc7_update(crc, byte); - } - (crc << 1) | 1 // shift left by 1 and set end bit - } - - /// Build the 6-byte SPI command packet - pub fn to_spi_bytes(&self) -> [u8; 6] { - let crc = self.crc7(); - let token = 0x40 | (self.cmd & 0x3F); - let arg = self.arg.to_be_bytes(); - [token, arg[0], arg[1], arg[2], arg[3], crc] - } -} - -fn crc7_update(crc: u8, byte: u8) -> u8 { - let mut crc = crc; - let mut data = byte; - for _ in 0..8 { - crc <<= 1; - if (crc ^ data) & 0x80 != 0 { - crc ^= 0x89; - } - data <<= 1; - } - crc -} - // ── Standard SD/MMC Commands ───────────────────────────────────────── // ── Broadcast commands (bc: no response, bcr: response) ── @@ -408,8 +311,8 @@ mod tests { let cmd = cmd52(true, 1, false, 0x1_ABCD, 0x55); // write=1, function=001, raw=0, addr=0x1ABCD (bits 25:9), stuff=0, data=0x55 let expected = (1u32 << 31) | (1u32 << 28) | (0x1_ABCDu32 << 9) | 0x55; - assert_eq!(cmd.arg, expected); - assert_eq!(cmd.cmd, 52); + assert_eq!(cmd.argument, expected); + assert_eq!(cmd.index, 52); } #[test] @@ -417,21 +320,33 @@ mod tests { let cmd = cmd53(false, 2, true, 0x1_FFFF, true, 0x1FF); // write=0, function=010, block_mode=1, op_code=1, addr=0x1FFFF, count=0x1FF let expected = (2u32 << 28) | (1u32 << 27) | (1u32 << 26) | (0x1_FFFFu32 << 9) | 0x1FF; - assert_eq!(cmd.arg, expected); - assert_eq!(cmd.cmd, 53); + assert_eq!(cmd.argument, expected); + assert_eq!(cmd.index, 53); } #[test] fn data_direction_classifies_block_commands() { - assert_eq!(cmd17(0).data_direction(), DataDirection::Read); - assert_eq!(cmd18(0).data_direction(), DataDirection::Read); - assert_eq!(cmd24(0).data_direction(), DataDirection::Write); - assert_eq!(cmd25(0).data_direction(), DataDirection::Write); + assert_eq!( + cmd17(0).data_direction(), + Some(sdio_host2::DataDirection::Read) + ); + assert_eq!( + cmd18(0).data_direction(), + Some(sdio_host2::DataDirection::Read) + ); + assert_eq!( + cmd24(0).data_direction(), + Some(sdio_host2::DataDirection::Write) + ); + assert_eq!( + cmd25(0).data_direction(), + Some(sdio_host2::DataDirection::Write) + ); // CMD6 is overloaded (ACMD6 vs SWITCH_FUNC); drivers tell the host // explicitly rather than relying on the index alone. - assert_eq!(cmd6(0).data_direction(), DataDirection::None); - assert_eq!(CMD0.data_direction(), DataDirection::None); - assert_eq!(CMD12.data_direction(), DataDirection::None); + assert_eq!(cmd6(0).data_direction(), None); + assert_eq!(CMD0.data_direction(), None); + assert_eq!(CMD12.data_direction(), None); assert!(CMD0.data_direction().is_none()); } @@ -449,35 +364,35 @@ mod tests { #[test] fn cmd6_high_speed_arg_matches_spec() { let switch = cmd6_high_speed(true); - assert_eq!(switch.cmd, 6); - assert_eq!(switch.arg, 0x80FF_FFF1); + assert_eq!(switch.index, 6); + assert_eq!(switch.argument, 0x80FF_FFF1); let check = cmd6_high_speed(false); - assert_eq!(check.arg, 0x00FF_FFF1); + assert_eq!(check.argument, 0x00FF_FFF1); } #[test] fn cmd6_sd_access_mode_arg_selects_group1_function() { let sdr104 = cmd6_sd_access_mode(true, 3); - assert_eq!(sdr104.cmd, 6); - assert_eq!(sdr104.arg, 0x80FF_FFF3); + assert_eq!(sdr104.index, 6); + assert_eq!(sdr104.argument, 0x80FF_FFF3); let ddr50 = cmd6_sd_access_mode(false, 4); - assert_eq!(ddr50.arg, 0x00FF_FFF4); + assert_eq!(ddr50.argument, 0x00FF_FFF4); } #[test] fn cmd41_with_s18r_sets_1v8_request_bit() { let cmd = cmd41_with_s18r(true, 0xFF8000, true); - assert_eq!(cmd.arg, 0x4100_0000 | 0x00FF_8000); + assert_eq!(cmd.argument, 0x4100_0000 | 0x00FF_8000); } #[test] fn with_resp_type_overrides_only_resp_type() { let original = cmd41(true, 0xFF8000); let overridden = original.with_resp_type(ResponseType::R1); - assert_eq!(overridden.cmd, original.cmd); - assert_eq!(overridden.arg, original.arg); - assert_eq!(overridden.resp_type, ResponseType::R1); - assert_eq!(original.resp_type, ResponseType::R3); + assert_eq!(overridden.index, original.index); + assert_eq!(overridden.argument, original.argument); + assert_eq!(overridden.response, ResponseType::R1); + assert_eq!(original.response, ResponseType::R3); } } diff --git a/drivers/blk/sdmmc-protocol/src/error.rs b/drivers/blk/sdmmc-protocol/src/error.rs index b113e7aec7..1ebb2ad53e 100644 --- a/drivers/blk/sdmmc-protocol/src/error.rs +++ b/drivers/blk/sdmmc-protocol/src/error.rs @@ -96,6 +96,8 @@ pub enum Error { Crc(ErrorContext), /// Card is not responding or not inserted. NoCard, + /// Host/controller currently has another active request. + Busy, /// Command index is not supported on this transport. UnsupportedCommand, /// Bad response received during the wrapped phase. @@ -226,6 +228,7 @@ impl fmt::Display for Error { Self::Timeout(ctx) => write!(f, "timeout during {ctx}"), Self::Crc(ctx) => write!(f, "CRC mismatch during {ctx}"), Self::NoCard => f.write_str("no card present"), + Self::Busy => f.write_str("host controller is busy"), Self::UnsupportedCommand => f.write_str("command not supported by transport"), Self::BadResponse(ctx) => write!(f, "bad response during {ctx}"), Self::CardError(err) => write!(f, "card reported {err}"), diff --git a/drivers/blk/sdmmc-protocol/src/lib.rs b/drivers/blk/sdmmc-protocol/src/lib.rs index 4971910d99..33ba20db28 100644 --- a/drivers/blk/sdmmc-protocol/src/lib.rs +++ b/drivers/blk/sdmmc-protocol/src/lib.rs @@ -81,6 +81,9 @@ #![no_std] +#[cfg(any(feature = "sdio", feature = "rdif"))] +extern crate alloc; + pub mod block; pub mod cmd; mod common; @@ -94,6 +97,9 @@ pub mod spi; #[cfg(feature = "sdio")] pub mod sdio; +#[cfg(feature = "rdif")] +pub mod rdif; + pub use block::{ BlockBufferConfig, BlockPoll, BlockRequestId, BlockTransferDirection, BlockTransferMode, BlockTransferState, CommandPoll, CommandResponsePoll, DataCommandDirection, DataCommandPoll, diff --git a/drivers/blk/sdmmc-protocol/src/rdif.rs b/drivers/blk/sdmmc-protocol/src/rdif.rs new file mode 100644 index 0000000000..c1b35b9652 --- /dev/null +++ b/drivers/blk/sdmmc-protocol/src/rdif.rs @@ -0,0 +1,1946 @@ +//! RDIF block-device bridge for SDIO-backed SD/MMC hosts. +//! +//! This module owns the reusable queue/runtime-independent part of adapting a +//! [`crate::sdio::SdioSdmmc`] card to [`rdif_block`]. Host crates provide the +//! small controller-specific [`BlockHost`] impl that submits and polls one +//! block request. + +use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; +use core::{ + cell::UnsafeCell, + marker::PhantomData, + num::NonZeroUsize, + ptr::NonNull, + sync::atomic::{AtomicBool, Ordering}, +}; + +use log::warn; +use rdif_block::dma_api::{CompletedDma, DeviceDma, PreparedDma}; +pub use rdif_block::{ + BInterface, BIrqHandler, BOwnedQueue, BQueue, BlkError, IQueue, IQueueOwned, Interface, + IrqHandlerHandle, IrqHandlerSlot, OwnedRequest, PollError, QueueHandle, Request, RequestId, + RequestPoll as OwnedRequestPoll, RequestStatus, SubmitError, dma_api, +}; + +use crate::{ + BlockPoll, BlockRequestId, BlockTransferMode, DataCommandPoll, Error, + sdio::{ + SdioHost, SdioHost2Adapter, SdioHost2DataRequest, SdioHost2Irq, SdioIrqHandle, SdioIrqHost, + SdioSdmmc, block_queue_ready_from_host_event, + }, +}; + +pub const BLOCK_SIZE: usize = 512; +pub const DEFAULT_DMA_MASK: u64 = u32::MAX as u64; +pub const DEFAULT_DMA_MAX_BLOCKS_PER_REQUEST: u32 = u16::MAX as u32 + 1; + +#[derive(Clone)] +pub struct BlockConfig { + pub name: &'static str, + pub capacity_blocks: u64, + pub dma_mask: u64, + pub dma_domain: dma_api::DmaDomainId, + pub max_blocks_per_request: u32, + pub max_segment_size: usize, + pub irq_driven: bool, + pub dma: Option, +} + +impl BlockConfig { + pub fn dma(name: &'static str, capacity_blocks: u64, irq_driven: bool, dma: DeviceDma) -> Self { + let dma_mask = dma.dma_mask(); + Self { + name, + capacity_blocks, + dma_mask, + dma_domain: dma.domain_id(), + max_blocks_per_request: DEFAULT_DMA_MAX_BLOCKS_PER_REQUEST, + max_segment_size: usize::MAX, + irq_driven, + dma: Some(dma), + } + } + + pub const fn fifo(name: &'static str, capacity_blocks: u64, irq_driven: bool) -> Self { + Self { + name, + capacity_blocks, + dma_mask: DEFAULT_DMA_MASK, + dma_domain: dma_api::DmaDomainId::legacy_global(), + max_blocks_per_request: 1, + max_segment_size: BLOCK_SIZE, + irq_driven, + dma: None, + } + } + + pub fn with_dma_mask(mut self, dma_mask: u64) -> Self { + self.dma_mask = dma_mask; + self + } + + pub fn with_max_blocks_per_request(mut self, max_blocks_per_request: u32) -> Self { + self.max_blocks_per_request = max_blocks_per_request; + self + } + + pub fn with_max_segment_size(mut self, max_segment_size: usize) -> Self { + self.max_segment_size = max_segment_size; + self + } + + pub fn with_irq_driven(mut self, irq_driven: bool) -> Self { + self.irq_driven = irq_driven; + self + } + + pub fn with_dma(mut self, dma: DeviceDma) -> Self { + self.dma_mask = dma.dma_mask(); + self.dma_domain = dma.domain_id(); + self.dma = Some(dma); + self + } + + pub const fn uses_dma(&self) -> bool { + self.dma.is_some() + } +} + +pub trait BlockHost: SdioIrqHost + Send + Sync + 'static { + type Request: Send + 'static; + type Slot: Default + Send + 'static; + + fn submit_read_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result; + + fn submit_write_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result; + + fn poll_block_request( + &mut self, + pending: &mut Option, + request: BlockRequestId, + slot: &mut Self::Slot, + ) -> Result; + + fn abort_request( + &mut self, + pending: &mut Option, + slot: &mut Self::Slot, + ) -> Result<(), Error>; + + fn request_id(request: &Self::Request) -> BlockRequestId; + + fn submit_owned_read_request( + &mut self, + _start_block: u32, + buffer: PreparedDma, + _slot: &mut Self::Slot, + _pending: &mut Option, + ) -> Result { + Err(OwnedBlockSubmitError::new(BlkError::NotSupported, buffer)) + } + + fn submit_owned_write_request( + &mut self, + _start_block: u32, + buffer: PreparedDma, + _slot: &mut Self::Slot, + _pending: &mut Option, + ) -> Result { + Err(OwnedBlockSubmitError::new(BlkError::NotSupported, buffer)) + } + + fn take_completed_dma(_slot: &mut Self::Slot) -> Option { + None + } +} + +pub struct OwnedBlockSubmitError { + error: BlkError, + buffer: Box, +} + +impl OwnedBlockSubmitError { + fn new(error: BlkError, buffer: PreparedDma) -> Self { + Self { + error, + buffer: Box::new(buffer), + } + } + + fn into_parts(self) -> (BlkError, PreparedDma) { + (self.error, *self.buffer) + } +} + +#[derive(Default)] +pub struct ProtocolBlockSlot { + next_id: usize, + active_id: Option, + completed_dma: Option, +} + +pub struct ProtocolBlockRequest<'a, H: SdioHost2Irq + 'static> { + id: BlockRequestId, + inner: SdioHost2DataRequest<'a, H>, +} + +// SAFETY: The request guard owns no shared access to the host; it forwards +// completion/abort through `SdioHost2Adapter`'s serialized shared core. +unsafe impl Send for ProtocolBlockRequest<'static, H> +where + H: SdioHost2Irq + Send + 'static, + H::TransactionRequest<'static>: Send, +{ +} + +impl BlockHost for SdioHost2Adapter +where + H: SdioHost2Irq + Send + 'static, + H::TransactionRequest<'static>: Send, +{ + type Request = ProtocolBlockRequest<'static, H>; + type Slot = ProtocolBlockSlot; + + fn submit_read_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + _dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_protocol_request(self, start_block, buffer, size, slot, pending, true) + } + + fn submit_write_request( + &mut self, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + _dma: Option<&DeviceDma>, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_protocol_request(self, start_block, buffer, size, slot, pending, false) + } + + fn poll_block_request( + &mut self, + pending: &mut Option, + request: BlockRequestId, + slot: &mut Self::Slot, + ) -> Result { + let Some(active) = pending.as_mut() else { + return Err(Error::InvalidArgument); + }; + if active.id != request { + return Ok(BlockPoll::Pending); + } + match self.poll_data_request(&mut active.inner) { + Err(err) => { + let abort = active.inner.abort(); + *pending = None; + slot.active_id = None; + if let Err(abort_err) = abort { + warn!( + "sdmmc rdif: abort after poll error reported recovery error: {abort_err:?}" + ); + } + Err(err) + } + Ok(DataCommandPoll::Pending) => Ok(BlockPoll::Pending), + Ok(DataCommandPoll::Complete(_)) => { + slot.completed_dma = active.inner.take_completed_dma(); + *pending = None; + slot.active_id = None; + Ok(BlockPoll::Complete) + } + } + } + + fn abort_request( + &mut self, + pending: &mut Option, + slot: &mut Self::Slot, + ) -> Result<(), Error> { + let result = if let Some(active) = pending.as_mut() { + active.inner.abort() + } else { + Ok(()) + }; + *pending = None; + slot.active_id = None; + result + } + + fn request_id(request: &Self::Request) -> BlockRequestId { + request.id + } + + fn submit_owned_read_request( + &mut self, + start_block: u32, + buffer: PreparedDma, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_owned_protocol_request(self, start_block, buffer, slot, pending, true) + } + + fn submit_owned_write_request( + &mut self, + start_block: u32, + buffer: PreparedDma, + slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + submit_owned_protocol_request(self, start_block, buffer, slot, pending, false) + } + + fn take_completed_dma(slot: &mut Self::Slot) -> Option { + slot.completed_dma.take() + } +} + +fn submit_protocol_request( + host: &mut SdioHost2Adapter, + start_block: u32, + buffer: NonNull, + size: NonZeroUsize, + slot: &mut ProtocolBlockSlot, + pending: &mut Option>, + read: bool, +) -> Result +where + H: SdioHost2Irq + Send + 'static, + H::TransactionRequest<'static>: Send, +{ + if pending.is_some() || slot.active_id.is_some() { + return Err(BlkError::Retry); + } + if !size.get().is_multiple_of(BLOCK_SIZE) { + return Err(BlkError::Other("buffer is not block aligned")); + } + let blocks = u32::try_from(size.get() / BLOCK_SIZE).map_err(|_| BlkError::InvalidRequest)?; + let id = BlockRequestId::new(slot.next_id); + slot.next_id = slot.next_id.wrapping_add(1); + let inner = if read { + let cmd = if blocks == 1 { + crate::cmd::cmd17(start_block) + } else { + crate::cmd::cmd18(start_block) + }; + let buf: &'static mut [u8] = + unsafe { core::slice::from_raw_parts_mut(buffer.as_ptr(), size.get()) }; + host.submit_read_data(&cmd, buf, BLOCK_SIZE as u32, blocks) + .map_err(map_dev_err_to_blk_err)? + } else { + let cmd = if blocks == 1 { + crate::cmd::cmd24(start_block) + } else { + crate::cmd::cmd25(start_block) + }; + let buf: &'static [u8] = + unsafe { core::slice::from_raw_parts(buffer.as_ptr(), size.get()) }; + host.submit_write_data(&cmd, buf, BLOCK_SIZE as u32, blocks) + .map_err(map_dev_err_to_blk_err)? + }; + slot.active_id = Some(id); + *pending = Some(ProtocolBlockRequest { id, inner }); + Ok(id) +} + +fn submit_owned_protocol_request( + host: &mut SdioHost2Adapter, + start_block: u32, + buffer: PreparedDma, + slot: &mut ProtocolBlockSlot, + pending: &mut Option>, + read: bool, +) -> Result +where + H: SdioHost2Irq + Send + 'static, + H::TransactionRequest<'static>: Send, +{ + if pending.is_some() || slot.active_id.is_some() { + return Err(OwnedBlockSubmitError::new(BlkError::Retry, buffer)); + } + if !buffer.len().get().is_multiple_of(BLOCK_SIZE) { + return Err(OwnedBlockSubmitError::new( + BlkError::Other("buffer is not block aligned"), + buffer, + )); + } + let blocks = match u32::try_from(buffer.len().get() / BLOCK_SIZE) { + Ok(blocks) => blocks, + Err(_) => return Err(OwnedBlockSubmitError::new(BlkError::InvalidRequest, buffer)), + }; + let id = BlockRequestId::new(slot.next_id); + slot.next_id = slot.next_id.wrapping_add(1); + let cmd = if read { + if blocks == 1 { + crate::cmd::cmd17(start_block) + } else { + crate::cmd::cmd18(start_block) + } + } else if blocks == 1 { + crate::cmd::cmd24(start_block) + } else { + crate::cmd::cmd25(start_block) + }; + let direction = if read { + sdio_host2::DataDirection::Read + } else { + sdio_host2::DataDirection::Write + }; + let inner = match host.submit_dma_data(&cmd, direction, buffer, BLOCK_SIZE as u32, blocks) { + Ok(inner) => inner, + Err(err) => { + return Err(OwnedBlockSubmitError::new( + map_dev_err_to_blk_err(err.error), + err.into_buffer(), + )); + } + }; + slot.active_id = Some(id); + *pending = Some(ProtocolBlockRequest { id, inner }); + Ok(id) +} + +pub struct BlockDevice +where + H: BlockHost, +{ + control: Arc>, +} + +struct BlockControl +where + H: BlockHost, +{ + raw: SharedCore>, + config: BlockConfig, + irq_enabled: AtomicBool, + queue_taken: AtomicBool, + irq_handler: rdif_block::IrqHandlerSlot, +} + +impl BlockDevice +where + H: BlockHost, +{ + pub fn new(card: SdioSdmmc, config: BlockConfig) -> Self { + let raw = SharedCore::new(card); + let irq_handle = raw.with_mut(|raw| raw.host().irq_handle()); + let irq_handler = rdif_block::IrqHandlerSlot::new(Box::new(BlockIrqHandler:: { + handle: irq_handle, + _marker: PhantomData, + })); + Self { + control: Arc::new(BlockControl { + raw, + config, + irq_enabled: AtomicBool::new(false), + queue_taken: AtomicBool::new(false), + irq_handler, + }), + } + } + + pub fn config(&self) -> &BlockConfig { + &self.control.config + } + + fn queue_limits_with_mask(&self, dma_mask: u64) -> rdif_block::QueueLimits { + queue_limits(&self.control.config, dma_mask) + } +} + +impl BlockControl +where + H: BlockHost, +{ + fn claim_queue(&self) -> bool { + self.queue_taken + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + + fn release_queue(&self) { + self.queue_taken.store(false, Ordering::Release); + } +} + +impl rdif_block::DriverGeneric for BlockDevice +where + H: BlockHost, +{ + fn name(&self) -> &str { + self.control.config.name + } +} + +impl Interface for BlockDevice +where + H: BlockHost, +{ + fn device_info(&self) -> rdif_block::DeviceInfo { + device_info(&self.control.config) + } + + fn queue_limits(&self) -> rdif_block::QueueLimits { + self.queue_limits_with_mask(self.control.config.dma_mask) + } + + fn create_queue(&mut self) -> Option> { + if self.control.config.uses_dma() || !self.control.claim_queue() { + return None; + } + Some(Box::new(BlockQueue::::new(Arc::clone(&self.control), 0)) as _) + } + + fn create_owned_queue(&mut self) -> Option { + if self.control.config.dma.is_none() || !self.control.claim_queue() { + return None; + } + Some(QueueHandle::new(Box::new(BlockQueue::::new( + Arc::clone(&self.control), + 0, + )))) + } + + fn enable_irq(&self) { + if !self.control.config.irq_driven { + self.control.irq_enabled.store(false, Ordering::Release); + return; + } + let mut enabled = false; + self.control.raw.with_mut(|raw| { + if let Err(err) = SdioHost::enable_completion_irq(raw.host_mut()) { + warn!( + "{}: enable completion IRQ failed: {:?}", + self.control.config.name, err + ); + return; + } + enabled = raw.host().completion_irq_enabled(); + }); + self.control.irq_enabled.store(enabled, Ordering::Release); + } + + fn disable_irq(&self) { + self.control.raw.with_mut(|raw| { + if let Err(err) = SdioHost::disable_completion_irq(raw.host_mut()) { + warn!( + "{}: disable completion IRQ failed: {:?}", + self.control.config.name, err + ); + } + }); + self.control.irq_enabled.store(false, Ordering::Release); + } + + fn is_irq_enabled(&self) -> bool { + self.control.irq_enabled.load(Ordering::Acquire) + } + + fn irq_sources(&self) -> rdif_block::IrqSourceList { + if !self.control.config.irq_driven { + return Vec::new(); + } + vec![rdif_block::IrqSourceInfo::legacy( + rdif_block::IdList::from_bits(1), + )] + } + + fn take_irq_handler(&mut self, source_id: usize) -> Option> { + if !self.control.config.irq_driven || source_id != 0 { + return None; + } + self.control + .irq_handler + .take() + .map(|handler| Box::new(handler) as Box) + } +} + +pub struct BlockQueue +where + H: BlockHost, +{ + control: Arc>, + id: usize, + slot: H::Slot, + pending: Option, + split_transfer: Option, + completed: Vec, + completed_owned: Vec, +} + +#[derive(Clone, Copy, Debug)] +enum SplitDirection { + Read, + Write, +} + +struct SplitTransfer { + direction: SplitDirection, + public_id: RequestId, + next_card_block: u32, + block_addr_step: u32, + buffer_addr: usize, + next_offset: usize, + remaining_blocks: u32, +} + +impl BlockQueue +where + H: BlockHost, +{ + fn new(control: Arc>, id: usize) -> Self { + Self { + control, + id, + slot: H::Slot::default(), + pending: None, + split_transfer: None, + completed: Vec::new(), + completed_owned: Vec::new(), + } + } + + fn queue_info(&self) -> rdif_block::QueueInfo { + rdif_block::IQueue::info(self) + } + + fn submit_request_inner(&mut self, request: Request<'_>) -> Result { + rdif_block::validate_request(self.queue_info(), &request)?; + self.reap_pending_request()?; + let raw = self.control.raw.clone(); + raw.with_mut(|raw| { + let start_block = block_addr_for_card(request.lba, raw.is_high_capacity())?; + let buffer = request + .segments + .first() + .copied() + .ok_or(BlkError::InvalidRequest)?; + if !buffer.len().is_multiple_of(BLOCK_SIZE) { + return Err(BlkError::Other("buffer is not block aligned")); + } + let ptr = NonNull::new(buffer.virt).ok_or(BlkError::Other("buffer pointer is null"))?; + let size = NonZeroUsize::new(buffer.len()).ok_or(BlkError::Other("buffer is empty"))?; + let dma = self.control.config.dma.as_ref(); + let id = match request.op { + rdif_block::RequestOp::Read + if should_split_fifo_request(dma, request.block_count) => + { + self.submit_split_transfer( + raw, + SplitDirection::Read, + start_block, + ptr, + request.block_count, + raw.is_high_capacity(), + )? + } + rdif_block::RequestOp::Write + if should_split_fifo_request(dma, request.block_count) => + { + self.submit_split_transfer( + raw, + SplitDirection::Write, + start_block, + ptr, + request.block_count, + raw.is_high_capacity(), + )? + } + rdif_block::RequestOp::Read => { + let id = H::submit_read_request( + raw.host_mut(), + start_block, + ptr, + size, + dma, + &mut self.slot, + &mut self.pending, + )?; + RequestId::new(usize::from(id)) + } + rdif_block::RequestOp::Write => H::submit_write_request( + raw.host_mut(), + start_block, + ptr, + size, + dma, + &mut self.slot, + &mut self.pending, + ) + .map(|id| RequestId::new(usize::from(id)))?, + rdif_block::RequestOp::Flush + | rdif_block::RequestOp::Discard + | rdif_block::RequestOp::WriteZeroes => return Err(BlkError::NotSupported), + }; + Ok(id) + }) + } + + fn poll_request_inner(&mut self, request: RequestId) -> Result { + if let Some(index) = self.completed.iter().position(|id| *id == request) { + self.completed.swap_remove(index); + return Ok(RequestStatus::Complete); + } + if self + .split_transfer + .as_ref() + .is_some_and(|split| split.public_id != request) + { + return Ok(RequestStatus::Pending); + } + if self.split_transfer.is_some() { + return self.poll_split_transfer(request); + } + self.poll_direct_request(request) + } + + fn poll_direct_request(&mut self, request: RequestId) -> Result { + self.poll_host_request(BlockRequestId::new(usize::from(request))) + } + + fn poll_host_request(&mut self, request: BlockRequestId) -> Result { + let raw = self.control.raw.clone(); + match raw.with_mut(|raw| { + H::poll_block_request(raw.host_mut(), &mut self.pending, request, &mut self.slot) + }) { + Ok(BlockPoll::Complete) => Ok(RequestStatus::Complete), + Ok(BlockPoll::Pending) => Ok(RequestStatus::Pending), + Err(err) => Err(map_dev_err_to_blk_err(err)), + } + } + + fn submit_split_transfer( + &mut self, + raw: &mut SdioSdmmc, + direction: SplitDirection, + start_block: u32, + buffer: NonNull, + block_count: u32, + high_capacity: bool, + ) -> Result { + if self.pending.is_some() || self.split_transfer.is_some() { + return Err(BlkError::Retry); + } + let id = self.submit_split_child(raw, direction, start_block, buffer)?; + let public_id = RequestId::new(usize::from(id)); + let block_addr_step = if high_capacity { 1 } else { BLOCK_SIZE as u32 }; + let remaining_blocks = block_count - 1; + let next_card_block = if remaining_blocks == 0 { + start_block + } else { + start_block + .checked_add(block_addr_step) + .ok_or(BlkError::InvalidRequest)? + }; + self.split_transfer = Some(SplitTransfer { + direction, + public_id, + next_card_block, + block_addr_step, + buffer_addr: buffer.as_ptr() as usize, + next_offset: BLOCK_SIZE, + remaining_blocks, + }); + Ok(public_id) + } + + fn submit_split_child( + &mut self, + raw: &mut SdioSdmmc, + direction: SplitDirection, + card_block: u32, + buffer: NonNull, + ) -> Result { + let block_size = NonZeroUsize::new(BLOCK_SIZE).ok_or(BlkError::InvalidRequest)?; + match direction { + SplitDirection::Read => H::submit_read_request( + raw.host_mut(), + card_block, + buffer, + block_size, + None, + &mut self.slot, + &mut self.pending, + ), + SplitDirection::Write => H::submit_write_request( + raw.host_mut(), + card_block, + buffer, + block_size, + None, + &mut self.slot, + &mut self.pending, + ), + } + } + + fn poll_split_transfer(&mut self, request: RequestId) -> Result { + let child = self.pending_id().ok_or(BlkError::InvalidRequest)?; + let status = match self.poll_host_request(child) { + Ok(status) => status, + Err(err) => { + self.split_transfer = None; + return Err(err); + } + }; + match status { + RequestStatus::Pending => Ok(RequestStatus::Pending), + RequestStatus::Complete => self.advance_split_transfer(request), + } + } + + fn advance_split_transfer(&mut self, request: RequestId) -> Result { + if self + .split_transfer + .as_ref() + .is_none_or(|split| split.remaining_blocks == 0) + { + self.split_transfer = None; + return Ok(RequestStatus::Complete); + } + + let mut split = self.split_transfer.take().ok_or(BlkError::InvalidRequest)?; + if split.public_id != request { + self.split_transfer = Some(split); + return Ok(RequestStatus::Pending); + } + let ptr = split + .buffer_addr + .checked_add(split.next_offset) + .and_then(|addr| NonNull::new(addr as *mut u8)) + .ok_or(BlkError::Other("buffer pointer is null"))?; + let raw = self.control.raw.clone(); + let submit = raw.with_mut(|raw| { + self.submit_split_child(raw, split.direction, split.next_card_block, ptr) + }); + submit?; + split.remaining_blocks -= 1; + split.next_offset += BLOCK_SIZE; + if split.remaining_blocks > 0 { + split.next_card_block = split + .next_card_block + .checked_add(split.block_addr_step) + .ok_or(BlkError::InvalidRequest)?; + } + self.split_transfer = Some(split); + Ok(RequestStatus::Pending) + } + + fn pending_id(&self) -> Option { + self.pending.as_ref().map(H::request_id) + } + + fn active_request_id(&self) -> Option { + self.split_transfer + .as_ref() + .map(|split| split.public_id) + .or_else(|| { + self.pending + .as_ref() + .map(|pending| RequestId::new(usize::from(H::request_id(pending)))) + }) + } + + fn reap_pending_request(&mut self) -> Result { + let Some(active) = self.active_request_id() else { + return Ok(RequestStatus::Complete); + }; + match self.poll_request_inner(active) { + Ok(RequestStatus::Complete) => { + self.completed.push(active); + Ok(RequestStatus::Complete) + } + Ok(RequestStatus::Pending) => Err(BlkError::Retry), + Err(err) => Err(err), + } + } + + fn submit_owned_request_inner( + &mut self, + request: OwnedRequest, + ) -> Result { + if self.control.config.dma.is_none() { + return Err(SubmitError::new(BlkError::NotSupported, request)); + } + if self.split_transfer.is_some() || !self.completed_owned.is_empty() { + return Err(SubmitError::new(BlkError::Retry, request)); + } + if let Err(err) = rdif_block::validate_owned_request(self.queue_info(), &request) { + return Err(SubmitError::new(err, request)); + } + if let Some(active) = self + .pending + .as_ref() + .map(|pending| RequestId::new(usize::from(H::request_id(pending)))) + { + match self.poll_owned_request_inner(active) { + Ok(OwnedRequestPoll::Ready(completed)) => { + self.completed_owned.push(completed); + return Err(SubmitError::new(BlkError::Retry, request)); + } + Ok(OwnedRequestPoll::Pending) => { + return Err(SubmitError::new(BlkError::Retry, request)); + } + Err(_) => return Err(SubmitError::new(BlkError::Io, request)), + } + } + + let OwnedRequest { + op, + lba, + block_count, + data, + flags, + } = request; + let Some(buffer) = data else { + return Err(SubmitError::new( + BlkError::InvalidRequest, + OwnedRequest { + op, + lba, + block_count, + data: None, + flags, + }, + )); + }; + let raw = self.control.raw.clone(); + match raw.with_mut(|raw| { + let start_block = match block_addr_for_card(lba, raw.is_high_capacity()) { + Ok(start_block) => start_block, + Err(err) => return Err(OwnedBlockSubmitError::new(err, buffer)), + }; + match op { + rdif_block::RequestOp::Read => H::submit_owned_read_request( + raw.host_mut(), + start_block, + buffer, + &mut self.slot, + &mut self.pending, + ), + rdif_block::RequestOp::Write => H::submit_owned_write_request( + raw.host_mut(), + start_block, + buffer, + &mut self.slot, + &mut self.pending, + ), + rdif_block::RequestOp::Flush + | rdif_block::RequestOp::Discard + | rdif_block::RequestOp::WriteZeroes => { + Err(OwnedBlockSubmitError::new(BlkError::NotSupported, buffer)) + } + } + }) { + Ok(id) => Ok(RequestId::new(usize::from(id))), + Err(err) => { + let (error, buffer) = err.into_parts(); + Err(SubmitError::new( + error, + OwnedRequest { + op, + lba, + block_count, + data: Some(buffer), + flags, + }, + )) + } + } + } + + fn poll_owned_request_inner( + &mut self, + request: RequestId, + ) -> Result { + if let Some(index) = self + .completed_owned + .iter() + .position(|completed| completed.id == request) + { + return Ok(OwnedRequestPoll::Ready( + self.completed_owned.swap_remove(index), + )); + } + if self.split_transfer.is_some() { + return Err(PollError::WrongQueue); + } + let id = BlockRequestId::new(usize::from(request)); + let Some(active) = self.pending.as_ref() else { + return Err(PollError::UnknownRequest); + }; + if H::request_id(active) != id { + return Ok(OwnedRequestPoll::Pending); + } + let raw = self.control.raw.clone(); + match raw.with_mut(|raw| { + H::poll_block_request(raw.host_mut(), &mut self.pending, id, &mut self.slot) + }) { + Ok(BlockPoll::Pending) => Ok(OwnedRequestPoll::Pending), + Ok(BlockPoll::Complete) => { + let completed_dma = H::take_completed_dma(&mut self.slot); + self.pending = None; + Ok(OwnedRequestPoll::Ready(rdif_block::CompletedRequest::new( + request, + Ok(()), + completed_dma, + ))) + } + Err(err) => { + let raw = self.control.raw.clone(); + let abort = raw.with_mut(|raw| { + H::abort_request(raw.host_mut(), &mut self.pending, &mut self.slot) + }); + let completed_dma = H::take_completed_dma(&mut self.slot); + let result = match abort { + Ok(()) => Err(map_dev_err_to_blk_err(err)), + Err(recovery) => Err(map_dev_err_to_blk_err(recovery)), + }; + Ok(OwnedRequestPoll::Ready(rdif_block::CompletedRequest::new( + request, + result, + completed_dma, + ))) + } + } + } +} + +impl Drop for BlockQueue +where + H: BlockHost, +{ + fn drop(&mut self) { + if self.pending.is_some() { + let raw = self.control.raw.clone(); + raw.with_mut(|raw| { + if let Err(err) = + H::abort_request(raw.host_mut(), &mut self.pending, &mut self.slot) + { + warn!( + "sdmmc rdif: abort pending request on queue drop reported recovery error: \ + {err:?}" + ); + self.pending = None; + } + }); + } + self.split_transfer = None; + self.control.release_queue(); + } +} + +// SAFETY: `BlockQueue` owns one pending request slot. The concrete host +// request object owns any borrowed request segment until task-side poll +// reports completion or error. +unsafe impl IQueue for BlockQueue +where + H: BlockHost, +{ + fn id(&self) -> usize { + self.id + } + + fn info(&self) -> rdif_block::QueueInfo { + rdif_block::QueueInfo { + id: self.id, + device: device_info(&self.control.config), + limits: queue_limits(&self.control.config, self.control.config.dma_mask), + } + } + + fn submit_request(&mut self, request: Request<'_>) -> Result { + self.submit_request_inner(request) + } + + fn poll_request(&mut self, request: RequestId) -> Result { + self.poll_request_inner(request) + } +} + +impl IQueueOwned for BlockQueue +where + H: BlockHost, +{ + fn id(&self) -> usize { + self.id + } + + fn info(&self) -> rdif_block::QueueInfo { + rdif_block::QueueInfo { + id: self.id, + device: device_info(&self.control.config), + limits: queue_limits(&self.control.config, self.control.config.dma_mask), + } + } + + fn submit_request(&mut self, request: OwnedRequest) -> Result { + self.submit_owned_request_inner(request) + } + + fn poll_request(&mut self, request: RequestId) -> Result { + self.poll_owned_request_inner(request) + } + + fn cancel_request(&mut self, request: RequestId) -> Result { + if self + .pending + .as_ref() + .is_none_or(|pending| RequestId::new(usize::from(H::request_id(pending))) != request) + { + return Err(PollError::UnknownRequest); + } + let raw = self.control.raw.clone(); + let result = + raw.with_mut(|raw| H::abort_request(raw.host_mut(), &mut self.pending, &mut self.slot)); + let completed_dma = H::take_completed_dma(&mut self.slot); + let completion = match result { + Ok(()) => rdif_block::CompletedRequest::new(request, Err(BlkError::Io), completed_dma), + Err(err) => rdif_block::CompletedRequest::new( + request, + Err(map_dev_err_to_blk_err(err)), + completed_dma, + ), + }; + Ok(OwnedRequestPoll::Ready(completion)) + } + + fn shutdown(&mut self) { + if self.pending.is_some() { + let raw = self.control.raw.clone(); + raw.with_mut(|raw| { + if let Err(err) = + H::abort_request(raw.host_mut(), &mut self.pending, &mut self.slot) + { + warn!( + "sdmmc rdif: abort pending owned request on queue shutdown reported \ + recovery error: {err:?}" + ); + self.pending = None; + } + }); + } + } +} + +struct BlockIrqHandler +where + H: BlockHost, +{ + handle: ::IrqHandle, + _marker: PhantomData, +} + +impl rdif_block::IrqHandler for BlockIrqHandler +where + H: BlockHost, +{ + fn handle_irq(&self) -> rdif_block::Event { + let host_event = self.handle.handle_irq(); + let mut event = rdif_block::Event::none(); + if let Some(queue_id) = block_queue_ready_from_host_event(&host_event) { + event.push_queue(queue_id); + } + event + } +} + +pub fn queue_limits(config: &BlockConfig, dma_mask: u64) -> rdif_block::QueueLimits { + rdif_block::QueueLimits { + dma_mask, + dma_domain: config.dma_domain, + dma_alignment: BLOCK_SIZE, + max_inflight: 1, + max_blocks_per_request: config.max_blocks_per_request, + max_segments: 1, + max_segment_size: config.max_segment_size, + supported_flags: rdif_block::RequestFlags::NONE, + supports_flush: false, + supports_discard: false, + supports_write_zeroes: false, + } +} + +pub fn device_info(config: &BlockConfig) -> rdif_block::DeviceInfo { + rdif_block::DeviceInfo { + name: Some(config.name), + ..rdif_block::DeviceInfo::new(config.capacity_blocks, BLOCK_SIZE) + } +} + +pub fn block_addr_for_card(block_id: u64, high_capacity: bool) -> Result { + let block_id = u32::try_from(block_id).map_err(|_| BlkError::InvalidBlockIndex(block_id))?; + if high_capacity { + Ok(block_id) + } else { + block_id + .checked_mul(BLOCK_SIZE as u32) + .ok_or(BlkError::InvalidBlockIndex(block_id as u64)) + } +} + +pub fn map_dev_err_to_blk_err(err: Error) -> BlkError { + match err { + Error::Busy => BlkError::Retry, + Error::NoCard | Error::UnsupportedCommand | Error::CardLocked => BlkError::NotSupported, + Error::Misaligned | Error::InvalidArgument => { + BlkError::Other("SD/MMC request is not block aligned") + } + _ => BlkError::Io, + } +} + +pub fn transfer_mode_for_dma(dma: Option<&DeviceDma>) -> BlockTransferMode { + match dma { + Some(_) => BlockTransferMode::Dma, + None => BlockTransferMode::Fifo, + } +} + +fn should_split_fifo_request(dma: Option<&DeviceDma>, block_count: u32) -> bool { + dma.is_none() && block_count > 1 +} + +pub fn can_fallback_to_fifo(err: Error) -> bool { + matches!( + err, + Error::UnsupportedCommand | Error::InvalidArgument | Error::Misaligned + ) +} + +struct SharedCore { + inner: Arc>, +} + +struct SharedCoreInner { + value: UnsafeCell, + borrowed: AtomicBool, +} + +struct SharedCoreGuard<'a, T> { + inner: &'a SharedCoreInner, +} + +// SAFETY: `SharedCore` serializes all mutable access through a single atomic +// borrow flag. IRQ top halves use host-specific cloneable handles instead. +unsafe impl Send for SharedCoreInner {} + +// SAFETY: See the `Send` impl. +unsafe impl Sync for SharedCoreInner {} + +impl SharedCore { + fn new(value: T) -> Self { + Self { + inner: Arc::new(SharedCoreInner { + value: UnsafeCell::new(value), + borrowed: AtomicBool::new(false), + }), + } + } + + fn with_mut(&self, f: impl FnOnce(&mut T) -> R) -> R { + let mut guard = self.inner.enter(); + f(guard.get_mut()) + } +} + +impl Clone for SharedCore { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + +impl SharedCoreInner { + fn enter(&self) -> SharedCoreGuard<'_, T> { + loop { + if let Some(guard) = self.try_enter() { + return guard; + } + core::hint::spin_loop(); + } + } + + fn try_enter(&self) -> Option> { + self.borrowed + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .ok()?; + Some(SharedCoreGuard { inner: self }) + } +} + +impl SharedCoreGuard<'_, T> { + fn get_mut(&mut self) -> &mut T { + unsafe { &mut *self.inner.value.get() } + } +} + +impl Drop for SharedCoreGuard<'_, T> { + fn drop(&mut self) { + self.inner.borrowed.store(false, Ordering::Release); + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use sdio_host2::{RequestPoll, SdioHost as PhysicalSdioHost, Transaction}; + + use super::*; + use crate::{ + CommandResponsePoll, DataCommandPoll, OperationPoll, + cmd::Command, + sdio::{ClockSpeed, HostEvent, HostEventKind}, + }; + + fn block_control(config: BlockConfig) -> Arc> { + let raw = SharedCore::new(SdioSdmmc::new(MockHost::default())); + let irq_handle = raw.with_mut(|raw| raw.host().irq_handle()); + Arc::new(BlockControl { + raw, + config, + irq_enabled: AtomicBool::new(false), + queue_taken: AtomicBool::new(false), + irq_handler: rdif_block::IrqHandlerSlot::new(Box::new(BlockIrqHandler:: { + handle: irq_handle, + _marker: PhantomData, + })), + }) + } + + #[test] + fn fifo_config_limits_single_block_requests() { + let config = BlockConfig::fifo("test-sdmmc", 8, true); + let limits = queue_limits(&config, DEFAULT_DMA_MASK); + + assert_eq!(limits.max_inflight, 1); + assert_eq!(limits.max_blocks_per_request, 1); + assert_eq!(limits.max_segment_size, BLOCK_SIZE); + assert!(!limits.supports_flush); + } + + #[test] + fn disabled_irq_policy_does_not_advertise_sources() { + let device = BlockDevice::new( + SdioSdmmc::new(MockHost::default()), + BlockConfig::fifo("mock-sd", 8, false), + ); + + assert!(Interface::irq_sources(&device).is_empty()); + } + + #[test] + fn enabled_irq_handler_maps_host_event_to_queue_zero() { + let mut device = BlockDevice::new( + SdioSdmmc::new(MockHost::default()), + BlockConfig::fifo("mock-sd", 8, true), + ); + let handler = Interface::take_irq_handler(&mut device, 0).unwrap(); + + let event = handler.handle_irq(); + + assert!(event.queues.contains(0)); + assert!(!event.is_empty()); + } + + #[test] + fn dma_config_exposes_one_owned_queue_while_handle_is_live() { + let dma = DeviceDma::new_legacy(u32::MAX as u64, &TEST_DMA); + let mut device = BlockDevice::new( + SdioSdmmc::new(MockHost::default()), + BlockConfig::dma("mock-sd", 8, false, dma), + ); + + assert!(Interface::create_queue(&mut device).is_none()); + let queue = Interface::create_owned_queue(&mut device); + assert!(queue.is_some()); + assert!(Interface::create_owned_queue(&mut device).is_none()); + drop(queue); + assert!(Interface::create_owned_queue(&mut device).is_some()); + } + + #[test] + fn poll_request_only_completes_matching_request_id() { + let mut queue = + BlockQueue::::new(block_control(BlockConfig::fifo("mock-sd", 8, false)), 0); + queue.pending = Some(MockRequest { + id: BlockRequestId::new(7), + }); + + assert_eq!( + queue.poll_request_inner(RequestId::new(8)), + Ok(RequestStatus::Pending) + ); + assert_eq!( + queue.poll_request_inner(RequestId::new(7)), + Ok(RequestStatus::Complete) + ); + assert!(queue.pending.is_none()); + } + + #[test] + fn unsupported_ops_are_rejected() { + let mut queue = + BlockQueue::::new(block_control(BlockConfig::fifo("mock-sd", 8, false)), 0); + let mut segments = []; + let request = Request { + op: rdif_block::RequestOp::Flush, + lba: 0, + block_count: 0, + segments: &mut segments, + flags: rdif_block::RequestFlags::NONE, + }; + + assert_eq!( + rdif_block::IQueue::submit_request(&mut queue, request), + Err(BlkError::NotSupported) + ); + } + + #[test] + fn dropping_queue_aborts_pending_request() { + let control = block_control(BlockConfig::fifo("mock-sd", 8, false)); + let raw = control.raw.clone(); + let mut backing = [0u8; BLOCK_SIZE]; + let mut segments = + [ + unsafe { + rdif_block::Buffer::from_raw_parts(backing.as_mut_ptr(), 0, backing.len()) + }, + ]; + { + let mut queue = BlockQueue::::new(Arc::clone(&control), 0); + let request = Request { + op: rdif_block::RequestOp::Read, + lba: 0, + block_count: 1, + segments: &mut segments, + flags: rdif_block::RequestFlags::NONE, + }; + + rdif_block::IQueue::submit_request(&mut queue, request).unwrap(); + } + + assert_eq!( + raw.with_mut(|raw| raw.host().aborts.load(Ordering::Acquire)), + 1 + ); + } + + #[test] + fn host2_submit_failure_does_not_leak_active_slot() { + let mut host = SdioHost2Adapter::new(Host2BlockMock { + submit_error: Some(sdio_host2::Error::Busy), + ..Host2BlockMock::default() + }); + let mut slot = ProtocolBlockSlot::default(); + let mut pending = None; + let mut backing = [0u8; BLOCK_SIZE]; + let buffer = NonNull::new(backing.as_mut_ptr()).unwrap(); + let size = NonZeroUsize::new(BLOCK_SIZE).unwrap(); + + assert_eq!( + as BlockHost>::submit_read_request( + &mut host, + 0, + buffer, + size, + None, + &mut slot, + &mut pending, + ), + Err(BlkError::Retry) + ); + assert!(pending.is_none()); + assert!(slot.active_id.is_none()); + + as BlockHost>::submit_read_request( + &mut host, + 0, + buffer, + size, + None, + &mut slot, + &mut pending, + ) + .expect("slot should accept a new request after submit failure"); + } + + #[test] + fn host2_poll_error_clears_pending_and_active_slot() { + let mut host = SdioHost2Adapter::new(Host2BlockMock { + poll_error: Some(sdio_host2::Error::Timeout), + ..Host2BlockMock::default() + }); + let mut slot = ProtocolBlockSlot::default(); + let mut pending = None; + let mut backing = [0u8; BLOCK_SIZE]; + let buffer = NonNull::new(backing.as_mut_ptr()).unwrap(); + let size = NonZeroUsize::new(BLOCK_SIZE).unwrap(); + let id = as BlockHost>::submit_read_request( + &mut host, + 0, + buffer, + size, + None, + &mut slot, + &mut pending, + ) + .unwrap(); + + assert!(matches!( + as BlockHost>::poll_block_request( + &mut host, + &mut pending, + id, + &mut slot, + ), + Err(Error::Timeout(_)) + )); + assert!(pending.is_none()); + assert!(slot.active_id.is_none()); + } + + #[derive(Clone, Default)] + struct MockIrqHandle; + + impl SdioIrqHandle for MockIrqHandle { + type Event = MockEvent; + + fn handle_irq(&self) -> Self::Event { + MockEvent(HostEventKind::TransferComplete) + } + } + + #[derive(Clone, Copy, Default)] + struct MockEvent(HostEventKind); + + impl HostEvent for MockEvent { + fn kind(&self) -> HostEventKind { + self.0 + } + } + + #[derive(Default)] + struct MockHost { + irq_enabled: AtomicBool, + next_id: AtomicUsize, + aborts: AtomicUsize, + read_sizes: Vec, + write_sizes: Vec, + } + + #[derive(Default)] + struct MockSlot; + + struct MockRequest { + id: BlockRequestId, + } + + struct TestDma; + static TEST_DMA: TestDma = TestDma; + + impl dma_api::DmaOp for TestDma { + fn page_size(&self) -> usize { + BLOCK_SIZE + } + + unsafe fn alloc_contiguous( + &self, + _constraints: dma_api::DmaConstraints, + _layout: core::alloc::Layout, + ) -> Option { + None + } + + unsafe fn dealloc_contiguous(&self, _handle: dma_api::DmaAllocHandle) {} + + unsafe fn alloc_coherent( + &self, + _constraints: dma_api::DmaConstraints, + _layout: core::alloc::Layout, + ) -> Option { + None + } + + unsafe fn dealloc_coherent(&self, _handle: dma_api::DmaAllocHandle) {} + + unsafe fn map_streaming( + &self, + _constraints: dma_api::DmaConstraints, + _addr: NonNull, + _size: NonZeroUsize, + _direction: dma_api::DmaDirection, + ) -> Result { + Err(dma_api::DmaError::NoMemory) + } + + unsafe fn unmap_streaming(&self, _handle: dma_api::DmaMapHandle) {} + } + + #[derive(Default)] + struct Host2BlockMock { + submit_error: Option, + poll_error: Option, + } + + struct Host2BlockRequest { + done: bool, + } + + impl PhysicalSdioHost for Host2BlockMock { + type TransactionRequest<'a> + = Host2BlockRequest + where + Self: 'a; + type BusRequest = Host2BlockRequest; + + unsafe fn submit_transaction<'a>( + &mut self, + _transaction: Transaction<'a>, + ) -> Result, sdio_host2::Error> + where + Self: 'a, + { + if let Some(err) = self.submit_error.take() { + return Err(err); + } + Ok(Host2BlockRequest { done: false }) + } + + fn poll_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result, sdio_host2::PollRequestError> + where + Self: 'a, + { + request.done = true; + if let Some(err) = self.poll_error.take() { + return Ok(RequestPoll::Ready(Err(err))); + } + Ok(RequestPoll::Ready(Ok(sdio_host2::RawResponse::empty()))) + } + + fn abort_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result<(), sdio_host2::Error> + where + Self: 'a, + { + request.done = true; + Ok(()) + } + + unsafe fn submit_bus_op( + &mut self, + _op: sdio_host2::BusOp, + ) -> Result { + Ok(Host2BlockRequest { done: false }) + } + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, sdio_host2::PollRequestError> { + request.done = true; + Ok(RequestPoll::Ready(Ok(()))) + } + + fn abort_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result<(), sdio_host2::Error> { + request.done = true; + Ok(()) + } + } + + impl SdioHost2Irq for Host2BlockMock { + type Event = MockEvent; + type IrqHandle = MockIrqHandle; + + fn irq_handle(&self) -> Self::IrqHandle { + MockIrqHandle + } + } + + impl SdioHost for MockHost { + type Event = MockEvent; + type DataRequest<'a> = (); + type BusRequest = crate::sdio::ReadyBusRequest; + + fn submit_command(&mut self, _cmd: &Command) -> Result<(), Error> { + Err(Error::UnsupportedCommand) + } + + fn poll_command_response(&mut self) -> Result { + Ok(CommandResponsePoll::Pending) + } + + fn submit_read_data<'a>( + &mut self, + _cmd: &Command, + _buf: &'a mut [u8], + _block_size: u32, + _block_count: u32, + ) -> Result, Error> { + Err(Error::UnsupportedCommand) + } + + fn submit_write_data<'a>( + &mut self, + _cmd: &Command, + _buf: &'a [u8], + _block_size: u32, + _block_count: u32, + ) -> Result, Error> { + Err(Error::UnsupportedCommand) + } + + fn poll_data_request<'a>( + &mut self, + _request: &mut Self::DataRequest<'a>, + ) -> Result { + Err(Error::UnsupportedCommand) + } + + fn set_bus_width(&mut self, _width: crate::sdio::BusWidth) -> Result<(), Error> { + Ok(()) + } + + fn set_clock(&mut self, _speed: ClockSpeed) -> Result<(), Error> { + Ok(()) + } + + fn submit_bus_op(&mut self, op: crate::sdio::SdioBusOp) -> Result { + crate::sdio::submit_ready_bus_op(self, op) + } + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, Error> { + crate::sdio::poll_ready_bus_op(request) + } + + fn enable_completion_irq(&mut self) -> Result<(), Error> { + self.irq_enabled.store(true, Ordering::Release); + Ok(()) + } + + fn disable_completion_irq(&mut self) -> Result<(), Error> { + self.irq_enabled.store(false, Ordering::Release); + Ok(()) + } + } + + impl SdioIrqHost for MockHost { + type IrqHandle = MockIrqHandle; + + fn irq_handle(&self) -> Self::IrqHandle { + MockIrqHandle + } + + fn completion_irq_enabled(&self) -> bool { + self.irq_enabled.load(Ordering::Acquire) + } + } + + impl BlockHost for MockHost { + type Request = MockRequest; + type Slot = MockSlot; + + fn submit_read_request( + &mut self, + _start_block: u32, + _buffer: NonNull, + _size: NonZeroUsize, + _dma: Option<&DeviceDma>, + _slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + self.read_sizes.push(_size.get()); + self.submit_mock_request(pending) + } + + fn submit_write_request( + &mut self, + _start_block: u32, + _buffer: NonNull, + _size: NonZeroUsize, + _dma: Option<&DeviceDma>, + _slot: &mut Self::Slot, + pending: &mut Option, + ) -> Result { + self.write_sizes.push(_size.get()); + self.submit_mock_request(pending) + } + + fn poll_block_request( + &mut self, + pending: &mut Option, + request: BlockRequestId, + _slot: &mut Self::Slot, + ) -> Result { + match pending.as_ref() { + Some(active) if active.id == request => { + *pending = None; + Ok(BlockPoll::Complete) + } + Some(_) => Ok(BlockPoll::Pending), + None => Ok(BlockPoll::Complete), + } + } + + fn abort_request( + &mut self, + pending: &mut Option, + _slot: &mut Self::Slot, + ) -> Result<(), Error> { + if pending.take().is_some() { + self.aborts.fetch_add(1, Ordering::AcqRel); + } + Ok(()) + } + + fn request_id(request: &Self::Request) -> BlockRequestId { + request.id + } + } + + impl MockHost { + fn submit_mock_request( + &self, + pending: &mut Option, + ) -> Result { + if pending.is_some() { + return Err(BlkError::Retry); + } + let id = BlockRequestId::new(self.next_id.fetch_add(1, Ordering::Relaxed)); + *pending = Some(MockRequest { id }); + Ok(id) + } + } + + #[test] + fn fifo_read_requests_are_split_to_single_blocks() { + let control = block_control( + BlockConfig::fifo("mock-sd", 16, false) + .with_max_blocks_per_request(8) + .with_max_segment_size(8 * BLOCK_SIZE), + ); + let raw = control.raw.clone(); + let mut queue = BlockQueue::::new(control, 0); + let mut backing = [0u8; 8 * BLOCK_SIZE]; + let mut segments = + [ + unsafe { + rdif_block::Buffer::from_raw_parts(backing.as_mut_ptr(), 0, backing.len()) + }, + ]; + let request = Request { + op: rdif_block::RequestOp::Read, + lba: 0, + block_count: 8, + segments: &mut segments, + flags: rdif_block::RequestFlags::NONE, + }; + + let id = rdif_block::IQueue::submit_request(&mut queue, request).unwrap(); + let mut polls = 0; + loop { + polls += 1; + match rdif_block::IQueue::poll_request(&mut queue, id) { + Ok(RequestStatus::Pending) => {} + Ok(RequestStatus::Complete) => break, + Err(err) => panic!("split read poll failed: {err:?}"), + } + } + assert_eq!(polls, 8); + + raw.with_mut(|raw| { + assert_eq!( + raw.host().read_sizes, + alloc::vec![BLOCK_SIZE; 8], + "FIFO read should avoid CMD18 multi-block requests on hosts without DMA" + ); + assert!(raw.host().write_sizes.is_empty()); + }); + } + + #[test] + fn fifo_write_requests_are_split_to_single_blocks() { + let control = block_control( + BlockConfig::fifo("mock-sd", 16, false) + .with_max_blocks_per_request(8) + .with_max_segment_size(8 * BLOCK_SIZE), + ); + let raw = control.raw.clone(); + let mut queue = BlockQueue::::new(control, 0); + let mut backing = [0u8; 8 * BLOCK_SIZE]; + let mut segments = + [ + unsafe { + rdif_block::Buffer::from_raw_parts(backing.as_mut_ptr(), 0, backing.len()) + }, + ]; + let request = Request { + op: rdif_block::RequestOp::Write, + lba: 0, + block_count: 8, + segments: &mut segments, + flags: rdif_block::RequestFlags::NONE, + }; + + let id = rdif_block::IQueue::submit_request(&mut queue, request).unwrap(); + let mut polls = 0; + loop { + polls += 1; + match rdif_block::IQueue::poll_request(&mut queue, id) { + Ok(RequestStatus::Pending) => {} + Ok(RequestStatus::Complete) => break, + Err(err) => panic!("split write poll failed: {err:?}"), + } + } + assert_eq!(polls, 8); + + raw.with_mut(|raw| { + assert!(raw.host().read_sizes.is_empty()); + assert_eq!( + raw.host().write_sizes, + alloc::vec![BLOCK_SIZE; 8], + "FIFO write should avoid CMD25/CMD12 multi-block requests on hosts without DMA" + ); + }); + } +} diff --git a/drivers/blk/sdmmc-protocol/src/response.rs b/drivers/blk/sdmmc-protocol/src/response.rs index 71b7733913..eb06ce6fbd 100644 --- a/drivers/blk/sdmmc-protocol/src/response.rs +++ b/drivers/blk/sdmmc-protocol/src/response.rs @@ -1,31 +1,6 @@ -use crate::error::{CardError, Error, ErrorContext, Phase}; +pub use sdio_host2::{RawResponse, ResponseType}; -/// SD/MMC response types -/// -/// Marked `#[non_exhaustive]`: new transport-level response shapes -/// (e.g. SDIO IO_RW extension, UHS-II) may be added before 1.0. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum ResponseType { - /// No response - None, - /// R1: Standard response (48-bit) - R1, - /// R1b: R1 with busy signal - R1b, - /// R2: CID/CSD register (136-bit) - R2, - /// R3: OCR register (48-bit) - R3, - /// R4: SDIO OCR (48-bit) - R4, - /// R5: SDIO RW (48-bit) - R5, - /// R6: Published RCA (48-bit, SD) - R6, - /// R7: Card interface condition (48-bit) - R7, -} +use crate::error::{CardError, Error, ErrorContext, Phase}; /// Parsed response from the card /// @@ -49,6 +24,51 @@ pub enum Response { R7(IfCondResponse), } +impl Response { + /// Convert a typed protocol response into the normalized physical response + /// words used by `sdio-host2`. + pub fn to_raw_response(self, expected: ResponseType) -> RawResponse { + let mut words = [0; 4]; + match self { + Self::Empty => {} + Self::R1(resp) | Self::R1b(resp) => words[0] = resp.raw, + Self::R2(bytes) => { + for (word, chunk) in words.iter_mut().zip(bytes.chunks_exact(4)) { + *word = u32::from_be_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + } + } + Self::R3(resp) => words[0] = resp.raw, + Self::R4(resp) => words[0] = resp.raw, + Self::R5(resp) => words[0] = resp.raw, + Self::R6(resp) => words[0] = resp.raw, + Self::R7(resp) => words[0] = resp.raw, + } + RawResponse::new(expected, words) + } +} + +/// Parse normalized physical response words into the protocol response type. +pub fn response_from_raw(raw: RawResponse) -> Result { + Ok(match raw.ty { + ResponseType::None => Response::Empty, + ResponseType::R1 => Response::R1(R1Response::from_native_raw(raw.words[0])?), + ResponseType::R1b => Response::R1b(R1Response::from_native_raw(raw.words[0])?), + ResponseType::R2 => { + let mut bytes = [0; 16]; + for (chunk, word) in bytes.chunks_exact_mut(4).zip(raw.words) { + chunk.copy_from_slice(&word.to_be_bytes()); + } + Response::R2(bytes) + } + ResponseType::R3 => Response::R3(OcrResponse::from_raw(raw.words[0])), + ResponseType::R4 => Response::R4(SdioOcrResponse::from_raw(raw.words[0])), + ResponseType::R5 => Response::R5(SdioRwResponse::from_raw(raw.words[0])), + ResponseType::R6 => Response::R6(RcaResponse::from_raw(raw.words[0])), + ResponseType::R7 => Response::R7(IfCondResponse::from_raw(raw.words[0])), + _ => return Err(Error::UnsupportedCommand), + }) +} + /// R1: Standard response — contains status bits #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct R1Response { diff --git a/drivers/blk/sdmmc-protocol/src/sdio.rs b/drivers/blk/sdmmc-protocol/src/sdio.rs index d3acd60132..fbe156d4d8 100644 --- a/drivers/blk/sdmmc-protocol/src/sdio.rs +++ b/drivers/blk/sdmmc-protocol/src/sdio.rs @@ -4,9 +4,21 @@ //! Implement [`SdioHost`] for your platform's SDIO peripheral; the host //! implementation controls command/data progress. -use core::task::Waker; +#[cfg(feature = "rdif")] +use alloc::boxed::Box; +use alloc::sync::Arc; +use core::{ + cell::UnsafeCell, + num::{NonZeroU16, NonZeroU32}, + sync::atomic::{AtomicBool, Ordering}, + task::Waker, +}; +use dma_api::CompletedDma; +#[cfg(feature = "rdif")] +use dma_api::PreparedDma; use log::{debug, info, warn}; +pub use sdio_host2::{BusWidth, ClockSpeed, SignalVoltage}; pub use crate::cmd::DataDirection; use crate::{ @@ -19,6 +31,26 @@ use crate::{ }, }; +#[cfg(feature = "rdif")] +pub(crate) struct DmaSubmitError { + pub error: Error, + buffer: Box, +} + +#[cfg(feature = "rdif")] +impl DmaSubmitError { + fn new(error: Error, buffer: PreparedDma) -> Self { + Self { + error, + buffer: Box::new(buffer), + } + } + + pub(crate) fn into_buffer(self) -> PreparedDma { + *self.buffer + } +} + /// Host IRQ event category returned by portable controller cores. /// /// Marked `#[non_exhaustive]`: new event categories (e.g. card-detect, @@ -122,70 +154,6 @@ pub fn block_queue_ready_from_host_event(event: &impl HostEvent) -> Option ...` arm. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum BusWidth { - /// 1-bit bus - Bit1, - /// 4-bit bus - Bit4, - /// 8-bit bus (eMMC). Configured via the MMC `CMD6 SWITCH` flow which is - /// outside the scope of the SD ACMD6 path used by this driver. - Bit8, -} - -/// SDIO clock speed -/// -/// Marked `#[non_exhaustive]`: HS400 / HS400_ES / SD Express modes are -/// expected to land before 1.0; downstream match sites must keep a -/// `_ => ...` arm. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum ClockSpeed { - /// Identification clock used during card reset / OCR negotiation. - Identification, - /// Default speed: up to 25 MHz - Default, - /// High speed: up to 50 MHz - HighSpeed, - /// SDR12: 12.5 MB/s - Sdr12, - /// SDR25: 25 MB/s - Sdr25, - /// SDR50: 50 MB/s - Sdr50, - /// SDR104: 104 MB/s - Sdr104, - /// DDR50: 50 MB/s (DDR) - Ddr50, - /// HS200: 200 MHz SDR, eMMC HS200 mode. Distinct from SDR104 - /// because the host typically routes eMMC and SD UHS-I through - /// different timing tables. - Hs200, -} - -/// Bus signaling voltage. Default-speed and HS modes use 3.3 V; UHS-I -/// and HS200/HS400 require switching to 1.8 V via CMD11. -/// -/// Marked `#[non_exhaustive]`: SD Express may introduce additional voltage -/// domains; downstream match sites must keep a `_ => ...` arm. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum SignalVoltage { - /// 3.3 V (or 3.0 V — they share an IO domain on most controllers). - /// The bus comes up here at power-on. - V330, - /// 1.8 V — required for SDR50 / SDR104 / DDR50 / HS200 / HS400. - V180, - /// 1.2 V — only relevant on certain HS200_12V eMMC parts. Most - /// hosts don't implement it; treated as opt-in. - V120, -} - /// Trait that the platform must implement for the SDIO host controller. /// /// The driver tracks the published RCA itself, so host implementations no @@ -231,6 +199,8 @@ pub trait SdioHost { request: &mut Self::DataRequest<'a>, ) -> Result; + type BusRequest; + /// Set the bus width fn set_bus_width(&mut self, width: BusWidth) -> Result<(), Error>; @@ -255,15 +225,21 @@ pub trait SdioHost { /// index (CMD19 for SD UHS-I, CMD21 for eMMC HS200). The host is /// responsible for issuing tuning blocks in a loop, comparing /// against the expected pattern, and reporting back whether a - /// stable sampling phase was found. + /// stable sampling phase was found. `block_size` is the protocol tuning + /// pattern length: SD CMD19 is 64 bytes, MMC CMD21 is 64 bytes on 4-bit + /// buses and 128 bytes on 8-bit buses. /// /// Default returns `UnsupportedCommand`. Hosts that report success /// without actually tuning are silently lying to the caller — only /// implement this when the controller can validate the result. - fn execute_tuning(&mut self, _cmd_index: u8) -> Result<(), Error> { + fn execute_tuning(&mut self, _cmd_index: u8, _block_size: NonZeroU16) -> Result<(), Error> { Err(Error::UnsupportedCommand) } + fn submit_bus_op(&mut self, op: SdioBusOp) -> Result; + + fn poll_bus_op(&mut self, request: &mut Self::BusRequest) -> Result, Error>; + /// Route command/data completion and error status to the host IRQ line. /// /// Default is a no-op so polling-only hosts do not have to implement IRQ @@ -314,6 +290,642 @@ pub trait SdioHost { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SdioBusOp { + ResetAll, + PowerOn, + PowerOff, + SetBusWidth(BusWidth), + SetClock(ClockSpeed), + SwitchVoltage(SignalVoltage), + ExecuteTuning { + cmd_index: u8, + block_size: NonZeroU16, + }, +} + +#[derive(Debug, Clone, Copy)] +pub struct ReadyBusRequest; + +pub fn submit_ready_bus_op>( + host: &mut H, + op: SdioBusOp, +) -> Result { + match op { + SdioBusOp::ResetAll | SdioBusOp::PowerOn | SdioBusOp::PowerOff => {} + SdioBusOp::SetBusWidth(width) => host.set_bus_width(width)?, + SdioBusOp::SetClock(speed) => host.set_clock(speed)?, + SdioBusOp::SwitchVoltage(voltage) => host.switch_voltage(voltage)?, + SdioBusOp::ExecuteTuning { + cmd_index, + block_size, + } => host.execute_tuning(cmd_index, block_size)?, + } + Ok(ReadyBusRequest) +} + +pub fn poll_ready_bus_op(_request: &mut ReadyBusRequest) -> Result, Error> { + Ok(OperationPoll::Complete(())) +} + +/// Compatibility adapter that lets the SD/MMC card state machine run on a +/// physical [`sdio_host2::SdioHost`] implementation. +/// +/// The protocol-facing [`SdioHost`] trait is kept for existing callers. New +/// host crates can implement `sdio_host2::SdioHost` natively and pass the host +/// through this adapter. +pub struct SdioHost2Adapter { + core: Host2Shared, + command_request: Option>, +} + +/// IRQ-capable extension used by [`SdioHost2Adapter`]. +/// +/// `sdio-host2` intentionally does not define IRQ abstractions. This protocol +/// crate only needs a way to forward host-specific completion IRQ handles when +/// a physical host is wrapped for the legacy `SdioHost` card state machine. +pub trait SdioHost2Irq: sdio_host2::SdioHost { + type Event: HostEvent + Default; + type IrqHandle: SdioIrqHandle; + + fn irq_handle(&self) -> Self::IrqHandle; + + fn completion_irq_enabled(&self) -> bool { + false + } + + fn enable_completion_irq(&mut self) -> Result<(), Error> { + Ok(()) + } + + fn disable_completion_irq(&mut self) -> Result<(), Error> { + Ok(()) + } + + fn handle_irq(&mut self) -> Self::Event { + self.irq_handle().handle_irq() + } +} + +impl SdioHost2Irq for T +where + T: sdio_host2::SdioHost + SdioIrqHost, +{ + type Event = ::Event; + type IrqHandle = ::IrqHandle; + + fn irq_handle(&self) -> Self::IrqHandle { + SdioIrqHost::irq_handle(self) + } + + fn completion_irq_enabled(&self) -> bool { + SdioIrqHost::completion_irq_enabled(self) + } + + fn enable_completion_irq(&mut self) -> Result<(), Error> { + SdioHost::enable_completion_irq(self) + } + + fn disable_completion_irq(&mut self) -> Result<(), Error> { + SdioHost::disable_completion_irq(self) + } + + fn handle_irq(&mut self) -> Self::Event { + SdioHost::handle_irq(self) + } +} + +pub struct NoopSdioHost2IrqHandle(core::marker::PhantomData E>); + +impl Default for NoopSdioHost2IrqHandle { + fn default() -> Self { + Self(core::marker::PhantomData) + } +} + +impl Clone for NoopSdioHost2IrqHandle { + fn clone(&self) -> Self { + Self(core::marker::PhantomData) + } +} + +impl SdioIrqHandle for NoopSdioHost2IrqHandle +where + E: HostEvent + Default + 'static, +{ + type Event = E; + + fn handle_irq(&self) -> Self::Event { + E::default() + } +} + +struct Host2Shared { + inner: Arc>, +} + +struct Host2SharedInner { + host: UnsafeCell, + borrowed: AtomicBool, +} + +// SAFETY: Access to `host` is serialized by `borrowed`; the wrapper never +// hands out references that outlive a `with_*` call. +unsafe impl Send for Host2SharedInner {} +// SAFETY: See the `Send` impl. +unsafe impl Sync for Host2SharedInner {} + +impl Clone for Host2Shared { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl Host2Shared { + fn new(host: H) -> Self { + Self { + inner: Arc::new(Host2SharedInner { + host: UnsafeCell::new(host), + borrowed: AtomicBool::new(false), + }), + } + } + + fn with_ref(&self, f: impl FnOnce(&H) -> R) -> R { + self.borrow(|host| f(host)) + } + + fn with_mut(&self, f: impl FnOnce(&mut H) -> R) -> R { + self.borrow(|host| f(host)) + } + + fn borrow(&self, f: impl FnOnce(&mut H) -> R) -> R { + if self + .inner + .borrowed + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + panic!("sdio-host2 adapter host borrowed concurrently"); + } + struct BorrowGuard<'a>(&'a AtomicBool); + impl Drop for BorrowGuard<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } + } + let _guard = BorrowGuard(&self.inner.borrowed); + // SAFETY: the atomic guard above serializes access to the host. + f(unsafe { &mut *self.inner.host.get() }) + } +} + +impl SdioHost2Adapter { + pub fn new(host: H) -> Self { + Self { + core: Host2Shared::new(host), + command_request: None, + } + } + + pub fn with_host(&self, f: impl FnOnce(&H) -> R) -> R { + self.core.with_ref(f) + } + + fn drain_bus_op(&mut self, request: &mut SdioHost2BusRequest) -> Result<(), Error> { + for _ in 0..SDIO_HOST2_COMPAT_POLL_LIMIT { + match self.poll_bus_op(request)? { + OperationPoll::Pending => core::hint::spin_loop(), + OperationPoll::Complete(()) => return Ok(()), + } + } + request.abort()?; + Err(Error::Timeout(ErrorContext::new(Phase::Init))) + } +} + +const SDIO_HOST2_COMPAT_POLL_LIMIT: u32 = 1_000_000; + +// SAFETY: The physical host is only accessed through `Host2Shared`, which +// serializes mutable borrows. `command_request` is only touched through +// `&mut self` and is aborted in `Drop`. +unsafe impl Send for SdioHost2Adapter +where + H: SdioHost2Irq + Send + 'static, + H::TransactionRequest<'static>: Send, +{ +} + +// SAFETY: Shared references only expose IRQ forwarding and `with_host`, both +// mediated by `Host2Shared`. Request mutation still requires `&mut self`. +unsafe impl Sync for SdioHost2Adapter +where + H: SdioHost2Irq + Send + 'static, + H::TransactionRequest<'static>: Send, +{ +} + +impl Drop for SdioHost2Adapter { + fn drop(&mut self) { + let Some(mut request) = self.command_request.take() else { + return; + }; + let result = self + .core + .with_mut(|host| host.abort_transaction(&mut request)) + .map_err(host2_error); + if let Err(err) = result { + warn!( + "sdio-host2 adapter: abort pending command on drop reported recovery error: \ + {err:?}" + ); + } + } +} + +pub struct SdioHost2DataRequest<'a, H: SdioHost2Irq + 'static> { + core: Host2Shared, + inner: Option>, + completed_dma: Option, +} + +impl SdioHost2DataRequest<'_, H> { + pub(crate) fn abort(&mut self) -> Result<(), Error> { + let Some(mut request) = self.inner.take() else { + return Ok(()); + }; + let (result, completed_dma) = self.core.with_mut(|host| { + let result = host.abort_transaction(&mut request).map_err(host2_error); + let completed_dma = host.take_completed_dma(&mut request); + (result, completed_dma) + }); + self.completed_dma = completed_dma; + result + } + + #[cfg(feature = "rdif")] + pub(crate) fn take_completed_dma(&mut self) -> Option { + self.completed_dma.take() + } +} + +impl Drop for SdioHost2DataRequest<'_, H> { + fn drop(&mut self) { + if let Err(err) = self.abort() { + warn!( + "sdio-host2 adapter: abort pending data request on drop reported recovery error: \ + {err:?}" + ); + } + } +} + +pub struct SdioHost2BusRequest { + core: Host2Shared, + inner: Option, +} + +impl SdioHost2BusRequest { + fn abort(&mut self) -> Result<(), Error> { + let Some(mut request) = self.inner.take() else { + return Ok(()); + }; + self.core + .with_mut(|host| host.abort_bus_op(&mut request)) + .map_err(host2_error) + } +} + +impl Drop for SdioHost2BusRequest { + fn drop(&mut self) { + if let Err(err) = self.abort() { + warn!( + "sdio-host2 adapter: abort pending bus op on drop reported recovery error: {err:?}" + ); + } + } +} + +impl SdioHost for SdioHost2Adapter { + type Event = H::Event; + type DataRequest<'a> + = SdioHost2DataRequest<'a, H> + where + Self: 'a; + type BusRequest = SdioHost2BusRequest; + + fn submit_command(&mut self, cmd: &Command) -> Result<(), Error> { + if self.command_request.is_some() { + return Err(Error::Busy); + } + let request = self + .core + .with_mut(|host| unsafe { + host.submit_transaction(sdio_host2::Transaction::command(*cmd)) + }) + .map_err(host2_error)?; + self.command_request = Some(request); + Ok(()) + } + + fn poll_command_response(&mut self) -> Result { + let mut request = self.command_request.take().ok_or(Error::InvalidArgument)?; + match self + .core + .with_mut(|host| host.poll_transaction(&mut request)) + { + Ok(sdio_host2::RequestPoll::Pending) => { + self.command_request = Some(request); + Ok(CommandResponsePoll::Pending) + } + Ok(sdio_host2::RequestPoll::Ready(Ok(raw))) => { + crate::response::response_from_raw(raw).map(CommandResponsePoll::Complete) + } + Ok(sdio_host2::RequestPoll::Ready(Err(err))) => Err(host2_error(err)), + Err(err) => { + self.command_request = Some(request); + Err(host2_poll_error(err)) + } + } + } + + fn submit_read_data<'a>( + &mut self, + cmd: &Command, + buf: &'a mut [u8], + block_size: u32, + block_count: u32, + ) -> Result, Error> { + let data = sdio_host2::DataPhase::read( + nonzero_block_size(block_size)?, + nonzero_block_count(block_count)?, + buf, + ) + .map_err(host2_error)?; + let request = self + .core + .with_mut(|host| unsafe { + host.submit_transaction(sdio_host2::Transaction::with_data(*cmd, data)) + }) + .map_err(host2_error)?; + Ok(SdioHost2DataRequest { + core: self.core.clone(), + inner: Some(request), + completed_dma: None, + }) + } + + fn submit_write_data<'a>( + &mut self, + cmd: &Command, + buf: &'a [u8], + block_size: u32, + block_count: u32, + ) -> Result, Error> { + let data = sdio_host2::DataPhase::write( + nonzero_block_size(block_size)?, + nonzero_block_count(block_count)?, + buf, + ) + .map_err(host2_error)?; + let request = self + .core + .with_mut(|host| unsafe { + host.submit_transaction(sdio_host2::Transaction::with_data(*cmd, data)) + }) + .map_err(host2_error)?; + Ok(SdioHost2DataRequest { + core: self.core.clone(), + inner: Some(request), + completed_dma: None, + }) + } + + fn poll_data_request<'a>( + &mut self, + request: &mut Self::DataRequest<'a>, + ) -> Result { + let inner = request.inner.as_mut().ok_or(Error::InvalidArgument)?; + match request.core.with_mut(|host| host.poll_transaction(inner)) { + Ok(sdio_host2::RequestPoll::Pending) => Ok(DataCommandPoll::Pending), + Ok(sdio_host2::RequestPoll::Ready(Ok(raw))) => { + request.completed_dma = request + .inner + .as_mut() + .and_then(|inner| request.core.with_mut(|host| host.take_completed_dma(inner))); + request.inner = None; + crate::response::response_from_raw(raw).map(DataCommandPoll::Complete) + } + Ok(sdio_host2::RequestPoll::Ready(Err(err))) => { + request.completed_dma = request + .inner + .as_mut() + .and_then(|inner| request.core.with_mut(|host| host.take_completed_dma(inner))); + request.inner = None; + Err(host2_error(err)) + } + Err(err) => Err(host2_poll_error(err)), + } + } + + fn set_bus_width(&mut self, width: BusWidth) -> Result<(), Error> { + let mut request = self.submit_bus_op(SdioBusOp::SetBusWidth(width))?; + self.drain_bus_op(&mut request) + } + + fn set_clock(&mut self, speed: ClockSpeed) -> Result<(), Error> { + let mut request = self.submit_bus_op(SdioBusOp::SetClock(speed))?; + self.drain_bus_op(&mut request) + } + + fn switch_voltage(&mut self, voltage: SignalVoltage) -> Result<(), Error> { + let mut request = self.submit_bus_op(SdioBusOp::SwitchVoltage(voltage))?; + self.drain_bus_op(&mut request) + } + + fn execute_tuning(&mut self, cmd_index: u8, block_size: NonZeroU16) -> Result<(), Error> { + let mut request = self.submit_bus_op(SdioBusOp::ExecuteTuning { + cmd_index, + block_size, + })?; + self.drain_bus_op(&mut request) + } + + fn submit_bus_op(&mut self, op: SdioBusOp) -> Result { + let op = match op { + SdioBusOp::ResetAll => sdio_host2::BusOp::ResetAll, + SdioBusOp::PowerOn => sdio_host2::BusOp::PowerOn, + SdioBusOp::PowerOff => sdio_host2::BusOp::PowerOff, + SdioBusOp::SetBusWidth(width) => sdio_host2::BusOp::SetBusWidth(width), + SdioBusOp::SetClock(speed) => sdio_host2::BusOp::SetClock(speed), + SdioBusOp::SwitchVoltage(voltage) => sdio_host2::BusOp::SetSignalVoltage(voltage), + SdioBusOp::ExecuteTuning { + cmd_index, + block_size, + } => { + let command = Command::new(cmd_index, 0, ResponseType::R1); + sdio_host2::BusOp::ExecuteTuning { + command, + block_size, + } + } + }; + let inner = self + .core + .with_mut(|host| unsafe { host.submit_bus_op(op) }) + .map_err(host2_error)?; + Ok(SdioHost2BusRequest { + core: self.core.clone(), + inner: Some(inner), + }) + } + + fn poll_bus_op(&mut self, request: &mut Self::BusRequest) -> Result, Error> { + let inner = request.inner.as_mut().ok_or(Error::InvalidArgument)?; + match request.core.with_mut(|host| host.poll_bus_op(inner)) { + Ok(sdio_host2::RequestPoll::Pending) => Ok(OperationPoll::Pending), + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) => { + request.inner = None; + Ok(OperationPoll::Complete(())) + } + Ok(sdio_host2::RequestPoll::Ready(Err(err))) => { + request.inner = None; + Err(host2_error(err)) + } + Err(err) => Err(host2_poll_error(err)), + } + } + + fn enable_completion_irq(&mut self) -> Result<(), Error> { + self.core.with_mut(|host| host.enable_completion_irq()) + } + + fn disable_completion_irq(&mut self) -> Result<(), Error> { + self.core.with_mut(|host| host.disable_completion_irq()) + } + + fn handle_irq(&mut self) -> Self::Event { + self.core.with_mut(|host| host.handle_irq()) + } + + fn now_ms(&self) -> Option { + self.core.with_ref(|host| host.now_ms()) + } +} + +#[cfg(feature = "rdif")] +impl SdioHost2Adapter { + pub(crate) fn submit_dma_data( + &mut self, + cmd: &Command, + direction: sdio_host2::DataDirection, + buffer: PreparedDma, + block_size: u32, + block_count: u32, + ) -> Result, DmaSubmitError> { + let block_size = match nonzero_block_size(block_size) { + Ok(block_size) => block_size, + Err(err) => return Err(DmaSubmitError::new(err, buffer)), + }; + let block_count = match nonzero_block_count(block_count) { + Ok(block_count) => block_count, + Err(err) => return Err(DmaSubmitError::new(err, buffer)), + }; + let data = sdio_host2::DataPhase::dma(direction, block_size, block_count, buffer).map_err( + |err| { + let (error, buffer) = err.into_parts(); + DmaSubmitError::new(host2_error(error), buffer) + }, + )?; + let transaction = sdio_host2::Transaction::with_data(*cmd, data); + let request = self + .core + .with_mut(|host| unsafe { host.submit_transaction_owned(transaction) }); + match request { + Ok(request) => Ok(SdioHost2DataRequest { + core: self.core.clone(), + inner: Some(request), + completed_dma: None, + }), + Err(err) => { + let error = host2_error(err.error); + let Some(transaction) = err.into_transaction() else { + panic!("sdio-host2 DMA submit consumed owned transaction on failure"); + }; + let Some(buffer) = recover_dma_buffer(transaction) else { + panic!("sdio-host2 DMA submit failure did not return DMA buffer"); + }; + Err(DmaSubmitError::new(error, buffer)) + } + } + } +} + +#[cfg(feature = "rdif")] +fn recover_dma_buffer(transaction: sdio_host2::Transaction<'_>) -> Option { + match transaction.data?.buffer { + sdio_host2::DataBuffer::Dma(buffer) => Some(buffer), + sdio_host2::DataBuffer::Read(_) | sdio_host2::DataBuffer::Write(_) => None, + } +} + +impl SdioIrqHost for SdioHost2Adapter { + type IrqHandle = H::IrqHandle; + + fn irq_handle(&self) -> Self::IrqHandle { + self.core.with_ref(|host| host.irq_handle()) + } + + fn completion_irq_enabled(&self) -> bool { + self.core.with_ref(|host| host.completion_irq_enabled()) + } +} + +impl SdioSdmmc> { + pub fn new_host2(host: H) -> Self { + Self::new(SdioHost2Adapter::new(host)) + } +} + +fn nonzero_block_size(block_size: u32) -> Result { + u16::try_from(block_size) + .ok() + .and_then(NonZeroU16::new) + .ok_or(Error::InvalidArgument) +} + +fn nonzero_block_count(block_count: u32) -> Result { + NonZeroU32::new(block_count).ok_or(Error::InvalidArgument) +} + +fn host2_error(err: sdio_host2::Error) -> Error { + match err { + sdio_host2::Error::Busy => Error::Busy, + sdio_host2::Error::Timeout => Error::Timeout(ErrorContext::default()), + sdio_host2::Error::Crc => Error::Crc(ErrorContext::default()), + sdio_host2::Error::NoCard => Error::NoCard, + sdio_host2::Error::Unsupported => Error::UnsupportedCommand, + sdio_host2::Error::InvalidArgument => Error::InvalidArgument, + sdio_host2::Error::Misaligned => Error::Misaligned, + sdio_host2::Error::Bus => Error::BusError(ErrorContext::default()), + sdio_host2::Error::Controller => Error::BusError(ErrorContext::default()), + _ => Error::BusError(ErrorContext::default()), + } +} + +fn host2_poll_error(err: sdio_host2::PollRequestError) -> Error { + match err { + sdio_host2::PollRequestError::AlreadyCompleted => Error::InvalidArgument, + sdio_host2::PollRequestError::WrongOwner + | sdio_host2::PollRequestError::WrongKind + | sdio_host2::PollRequestError::StaleGeneration + | sdio_host2::PollRequestError::RecoveryFailed => Error::BusError(ErrorContext::default()), + _ => Error::BusError(ErrorContext::default()), + } +} + /// Poll-cadence contract for [`SdioSdmmc::poll_init_request`] and the /// [`MmcSwitchRequest`] busy-wait sub-state. /// @@ -604,6 +1216,7 @@ pub struct SdioInitRequest<'a, H: SdioHost + 'a> { mmc_switch_request: Option, status_request: Option, command_request: Option, + bus_request: Option, current_bus_width: BusWidth, current_access_mode: Option, sd_access_index: usize, @@ -614,7 +1227,7 @@ pub struct SdioInitRequest<'a, H: SdioHost + 'a> { impl<'a, H: SdioHost + 'a> SdioInitRequest<'a, H> { fn new(preference: CardInitPreference, scratch: &'a mut SdioInitScratch) -> Self { Self { - state: SdioInitState::PollCmd0, + state: SdioInitState::ResetHost, preference, sd_v2: false, kind: None, @@ -635,6 +1248,7 @@ impl<'a, H: SdioHost + 'a> SdioInitRequest<'a, H> { mmc_switch_request: None, status_request: None, command_request: None, + bus_request: None, current_bus_width: BusWidth::Bit1, current_access_mode: None, sd_access_index: 0, @@ -658,6 +1272,15 @@ impl<'a, H: SdioHost + 'a> SdioInitRequest<'a, H> { #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum SdioInitState { + ResetHost, + PollResetHost, + PowerOn, + PollPowerOn, + ResetVoltage, + PollResetVoltage, + ResetBusWidth, + ResetClock, + SubmitCmd0, PollCmd0, PollCmd8, PollAcmd41Cmd55, @@ -670,17 +1293,27 @@ enum SdioInitState { PollCmd7, PollSdBusWidthCmd55, PollSdBusWidthAcmd6, + PollSdHostBusWidth, FinishCardSetup, + PollSdDefaultClock, PollMmcExtCsd, PollMmcBusWidth, + PollMmcHostBusWidth, PrepareMmcSpeed, + PollMmcHs200VoltageSwitch, PollMmcHs200Switch, + PollMmcHs200Clock, + PollMmcHs200Tuning, PollMmcHs200Status, PollMmcHs52Switch, + PollMmcHighSpeedClock, PrepareSdSpeed, PollSdSwitchFunctionCheck, PollSdVoltageSwitch, + PollSdSignalVoltage, PollSdSetAccessMode, + PollSdClock, + PollSdTuning, PollSdStatus, Complete, } @@ -769,6 +1402,19 @@ impl SdioSdmmc { self.sd_uhs_selection_enabled = enabled; } + fn mmc_tuning_block_size(&self) -> Result { + let bytes = if matches!(self.bus_width, BusWidth::Bit8) { + crate::cmd::MMC_TUNING_BLOCK_SIZE_8BIT + } else { + crate::cmd::SD_TUNING_BLOCK_SIZE + }; + nonzero_block_size(bytes) + } + + fn sd_tuning_block_size(&self) -> Result { + nonzero_block_size(crate::cmd::SD_TUNING_BLOCK_SIZE) + } + /// Which card family the driver detected. Meaningful only after a /// successful [`init`](Self::init); defaults to [`CardKind::Sd`]. pub fn kind(&self) -> CardKind { @@ -1033,21 +1679,50 @@ impl SdioSdmmc { H: 'a, { debug!("sdio: init starting"); - // Best-effort cleanup of any state a previous, aborted init may have - // left on the host (e.g. UHS_MODE bits, 4-bit bus, 50 MHz clock). - // We can't propagate failures here without poisoning the caller's - // retry path — `set_*` calls below run with `?` so the actual - // identification-mode requirements are enforced. - self.abort_init(); - - self.host.set_bus_width(BusWidth::Bit1)?; - self.host.set_clock(ClockSpeed::Identification)?; - - debug!("sdio: CMD0 reset"); - self.host.submit_command(&crate::cmd::CMD0)?; Ok(SdioInitRequest::new(preference, scratch)) } + fn submit_init_bus_op<'a>( + &mut self, + request: &mut SdioInitRequest<'a, H>, + op: SdioBusOp, + next: SdioInitState, + ) -> Result, Error> { + debug!("sdio: submit bus op {:?}", op); + request.bus_request = Some(self.host.submit_bus_op(op)?); + request.state = next; + Ok(OperationPoll::Pending) + } + + fn poll_init_bus_op<'a>( + &mut self, + request: &mut SdioInitRequest<'a, H>, + ) -> Result, Error> { + let mut bus_request = request.bus_request.take().ok_or(Error::InvalidArgument)?; + match self.host.poll_bus_op(&mut bus_request) { + Ok(OperationPoll::Pending) => { + request.bus_request = Some(bus_request); + Ok(OperationPoll::Pending) + } + Ok(OperationPoll::Complete(())) => Ok(OperationPoll::Complete(())), + Err(err) => Err(err), + } + } + + fn poll_init_bus_op_then<'a>( + &mut self, + request: &mut SdioInitRequest<'a, H>, + complete: impl FnOnce( + &mut Self, + &mut SdioInitRequest<'a, H>, + ) -> Result, Error>, + ) -> Result, Error> { + match self.poll_init_bus_op(request)? { + OperationPoll::Pending => Ok(OperationPoll::Pending), + OperationPoll::Complete(()) => complete(self, request), + } + } + /// Advance a submitted initialization request without blocking. /// /// On any terminal `Err` the controller is reset back toward an @@ -1078,6 +1753,91 @@ impl SdioSdmmc { const MMC_ACCESS_MODE_MASK: u32 = 0x6000_0000; match request.state { + SdioInitState::ResetHost => { + match self.host.submit_bus_op(SdioBusOp::ResetAll) { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollResetHost; + } + Err(Error::UnsupportedCommand) => { + debug!("sdio: host does not support reset bus op"); + request.state = SdioInitState::PowerOn; + } + Err(err) => return Err(err), + } + Ok(OperationPoll::Pending) + } + SdioInitState::PollResetHost => match self.poll_init_bus_op(request)? { + OperationPoll::Pending => Ok(OperationPoll::Pending), + OperationPoll::Complete(()) => { + request.state = SdioInitState::PowerOn; + Ok(OperationPoll::Pending) + } + }, + SdioInitState::PowerOn => { + match self.host.submit_bus_op(SdioBusOp::PowerOn) { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollPowerOn; + } + Err(Error::UnsupportedCommand) => { + debug!("sdio: host does not support power-on bus op"); + request.state = SdioInitState::ResetVoltage; + } + Err(err) => return Err(err), + } + Ok(OperationPoll::Pending) + } + SdioInitState::PollPowerOn => match self.poll_init_bus_op(request)? { + OperationPoll::Pending => Ok(OperationPoll::Pending), + OperationPoll::Complete(()) => { + request.state = SdioInitState::ResetVoltage; + Ok(OperationPoll::Pending) + } + }, + SdioInitState::ResetVoltage => { + match self + .host + .submit_bus_op(SdioBusOp::SwitchVoltage(SignalVoltage::V330)) + { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollResetVoltage; + } + Err(Error::UnsupportedCommand) => { + debug!("sdio: host does not support voltage reset"); + request.state = SdioInitState::ResetBusWidth; + } + Err(err) => return Err(err), + } + Ok(OperationPoll::Pending) + } + SdioInitState::PollResetVoltage => match self.poll_init_bus_op(request)? { + OperationPoll::Pending => Ok(OperationPoll::Pending), + OperationPoll::Complete(()) => self.submit_init_bus_op( + request, + SdioBusOp::SetBusWidth(BusWidth::Bit1), + SdioInitState::ResetClock, + ), + }, + SdioInitState::ResetBusWidth => self.submit_init_bus_op( + request, + SdioBusOp::SetBusWidth(BusWidth::Bit1), + SdioInitState::ResetClock, + ), + SdioInitState::ResetClock => self.poll_init_bus_op_then(request, |driver, request| { + driver.submit_init_bus_op( + request, + SdioBusOp::SetClock(ClockSpeed::Identification), + SdioInitState::SubmitCmd0, + ) + }), + SdioInitState::SubmitCmd0 => self.poll_init_bus_op_then(request, |driver, request| { + debug!("sdio: CMD0 reset"); + driver.host.submit_command(&crate::cmd::CMD0)?; + request.state = SdioInitState::PollCmd0; + Ok(OperationPoll::Pending) + }), SdioInitState::PollCmd0 => match self.host.poll_command_response()? { CommandResponsePoll::Pending => Ok(OperationPoll::Pending), CommandResponsePoll::Complete(_) => { @@ -1326,26 +2086,27 @@ impl SdioSdmmc { }, SdioInitState::PollSdBusWidthAcmd6 => match self.host.poll_command_response()? { CommandResponsePoll::Pending => Ok(OperationPoll::Pending), - CommandResponsePoll::Complete(_) => { - self.host.set_bus_width(BusWidth::Bit4)?; - self.bus_width = BusWidth::Bit4; + CommandResponsePoll::Complete(_) => self.submit_init_bus_op( + request, + SdioBusOp::SetBusWidth(BusWidth::Bit4), + SdioInitState::PollSdHostBusWidth, + ), + }, + SdioInitState::PollSdHostBusWidth => { + self.poll_init_bus_op_then(request, |driver, request| { + driver.bus_width = BusWidth::Bit4; request.state = SdioInitState::FinishCardSetup; Ok(OperationPoll::Pending) - } - }, + }) + } SdioInitState::FinishCardSetup => { let kind = request.kind.ok_or(Error::InvalidArgument)?; match kind { - CardKind::Sd => { - self.host.set_clock(ClockSpeed::Default)?; - if self.sd_speed_selection_enabled { - request.state = SdioInitState::PrepareSdSpeed; - } else { - debug!("sdio: SD speed selection disabled; staying at default speed"); - request.state = SdioInitState::Complete; - } - Ok(OperationPoll::Pending) - } + CardKind::Sd => self.submit_init_bus_op( + request, + SdioBusOp::SetClock(ClockSpeed::Default), + SdioInitState::PollSdDefaultClock, + ), CardKind::Mmc => { debug!("sdio: read MMC EXT_CSD"); // SAFETY: the slot's debug_assert traps re-lending; the @@ -1359,6 +2120,17 @@ impl SdioSdmmc { } } } + SdioInitState::PollSdDefaultClock => { + self.poll_init_bus_op_then(request, |driver, request| { + if driver.sd_speed_selection_enabled { + request.state = SdioInitState::PrepareSdSpeed; + } else { + debug!("sdio: SD speed selection disabled; staying at default speed"); + request.state = SdioInitState::Complete; + } + Ok(OperationPoll::Pending) + }) + } SdioInitState::PollMmcExtCsd => { let ext_request = request .ext_csd_request @@ -1394,22 +2166,16 @@ impl SdioSdmmc { Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), Ok(OperationPoll::Complete(())) => { request.mmc_switch_request = None; - match self.host.set_bus_width(request.current_bus_width) { - Ok(()) => { - self.bus_width = request.current_bus_width; - request.state = SdioInitState::PrepareMmcSpeed; - Ok(OperationPoll::Pending) - } - Err(err) if matches!(request.current_bus_width, BusWidth::Bit8) => { - debug!("sdio: 8-bit refused ({:?}), trying 4-bit", err); - submit_mmc_bus_width_or_continue(self, request, BusWidth::Bit4) - } - Err(err) if matches!(request.current_bus_width, BusWidth::Bit4) => { - debug!("sdio: 4-bit refused ({:?}), staying at 1-bit", err); - request.state = SdioInitState::PrepareMmcSpeed; + match self + .host + .submit_bus_op(SdioBusOp::SetBusWidth(request.current_bus_width)) + { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollMmcHostBusWidth; Ok(OperationPoll::Pending) } - Err(err) => Err(err), + Err(err) => handle_mmc_host_bus_width_error(self, request, err), } } Err(err) if matches!(request.current_bus_width, BusWidth::Bit8) => { @@ -1426,6 +2192,21 @@ impl SdioSdmmc { Err(err) => Err(err), } } + SdioInitState::PollMmcHostBusWidth => { + let mut bus_request = request.bus_request.take().ok_or(Error::InvalidArgument)?; + match self.host.poll_bus_op(&mut bus_request) { + Ok(OperationPoll::Pending) => { + request.bus_request = Some(bus_request); + Ok(OperationPoll::Pending) + } + Ok(OperationPoll::Complete(())) => { + self.bus_width = request.current_bus_width; + request.state = SdioInitState::PrepareMmcSpeed; + Ok(OperationPoll::Pending) + } + Err(err) => handle_mmc_host_bus_width_error(self, request, err), + } + } SdioInitState::PrepareMmcSpeed => { let Some(csd) = request.parsed_ext_csd.as_ref() else { return Err(Error::InvalidArgument); @@ -1436,15 +2217,13 @@ impl SdioSdmmc { && dt.supports_hs200() { request.mmc_hs200_attempted = true; - match self.host.switch_voltage(SignalVoltage::V180) { - Ok(()) => { - let switch_request = self.submit_mmc_switch( - 0b11, - crate::cmd::ext_csd::HS_TIMING as u8, - 0x02, - )?; - request.mmc_switch_request = Some(switch_request); - request.state = SdioInitState::PollMmcHs200Switch; + match self + .host + .submit_bus_op(SdioBusOp::SwitchVoltage(SignalVoltage::V180)) + { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollMmcHs200VoltageSwitch; return Ok(OperationPoll::Pending); } // The host has no way to actually drive the IO rail @@ -1468,6 +2247,41 @@ impl SdioSdmmc { } Ok(OperationPoll::Pending) } + SdioInitState::PollMmcHs200VoltageSwitch => { + let Some(csd) = request.parsed_ext_csd.as_ref() else { + return Err(Error::InvalidArgument); + }; + let supports_hs52 = csd.device_type().supports_hs_52(); + match self.poll_init_bus_op(request) { + Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), + Ok(OperationPoll::Complete(())) => { + let switch_request = self.submit_mmc_switch( + 0b11, + crate::cmd::ext_csd::HS_TIMING as u8, + 0x02, + )?; + request.mmc_switch_request = Some(switch_request); + request.state = SdioInitState::PollMmcHs200Switch; + Ok(OperationPoll::Pending) + } + Err(err) => { + debug!("sdio: switch_voltage(V180) failed ({:?})", err); + self.rollback_to_hs_compat(); + if supports_hs52 { + let switch_request = self.submit_mmc_switch( + 0b11, + crate::cmd::ext_csd::HS_TIMING as u8, + 1, + )?; + request.mmc_switch_request = Some(switch_request); + request.state = SdioInitState::PollMmcHs52Switch; + } else { + request.state = SdioInitState::Complete; + } + Ok(OperationPoll::Pending) + } + } + } SdioInitState::PollMmcHs200Switch => { let switch_request = request .mmc_switch_request @@ -1477,15 +2291,18 @@ impl SdioSdmmc { Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), Ok(OperationPoll::Complete(())) => { request.mmc_switch_request = None; - if self.host.set_clock(ClockSpeed::Hs200).is_ok() - && self.host.execute_tuning(21).is_ok() + match self + .host + .submit_bus_op(SdioBusOp::SetClock(ClockSpeed::Hs200)) { - let status_request = self.submit_status()?; - request.status_request = Some(status_request); - request.state = SdioInitState::PollMmcHs200Status; - } else { - self.rollback_to_hs_compat(); - request.state = SdioInitState::PrepareMmcSpeed; + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollMmcHs200Clock; + } + Err(_) => { + self.rollback_to_hs_compat(); + request.state = SdioInitState::PrepareMmcSpeed; + } } Ok(OperationPoll::Pending) } @@ -1498,6 +2315,45 @@ impl SdioSdmmc { } } } + SdioInitState::PollMmcHs200Clock => match self.poll_init_bus_op(request) { + Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), + Ok(OperationPoll::Complete(())) => { + let block_size = self.mmc_tuning_block_size()?; + match self.host.submit_bus_op(SdioBusOp::ExecuteTuning { + cmd_index: 21, + block_size, + }) { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollMmcHs200Tuning; + } + Err(_) => { + self.rollback_to_hs_compat(); + request.state = SdioInitState::PrepareMmcSpeed; + } + } + Ok(OperationPoll::Pending) + } + Err(_) => { + self.rollback_to_hs_compat(); + request.state = SdioInitState::PrepareMmcSpeed; + Ok(OperationPoll::Pending) + } + }, + SdioInitState::PollMmcHs200Tuning => match self.poll_init_bus_op(request) { + Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), + Ok(OperationPoll::Complete(())) => { + let status_request = self.submit_status()?; + request.status_request = Some(status_request); + request.state = SdioInitState::PollMmcHs200Status; + Ok(OperationPoll::Pending) + } + Err(_) => { + self.rollback_to_hs_compat(); + request.state = SdioInitState::PrepareMmcSpeed; + Ok(OperationPoll::Pending) + } + }, SdioInitState::PollMmcHs200Status => { let status_request = request .status_request @@ -1528,10 +2384,19 @@ impl SdioSdmmc { Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), Ok(OperationPoll::Complete(())) => { request.mmc_switch_request = None; - if let Err(_e) = self.host.set_clock(ClockSpeed::HighSpeed) { - debug!("sdio: host refused HighSpeed clock ({:?})", _e); + match self + .host + .submit_bus_op(SdioBusOp::SetClock(ClockSpeed::HighSpeed)) + { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollMmcHighSpeedClock; + } + Err(_e) => { + debug!("sdio: host refused HighSpeed clock ({:?})", _e); + request.state = SdioInitState::Complete; + } } - request.state = SdioInitState::Complete; Ok(OperationPoll::Pending) } Err(_e) => { @@ -1541,7 +2406,23 @@ impl SdioSdmmc { Ok(OperationPoll::Pending) } } - } + } + SdioInitState::PollMmcHighSpeedClock => match self.poll_init_bus_op(request) { + Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), + Ok(OperationPoll::Complete(())) => { + info!( + "sdio: MMC speed selected HighSpeed bus_width={:?}", + self.bus_width + ); + request.state = SdioInitState::Complete; + Ok(OperationPoll::Pending) + } + Err(_e) => { + debug!("sdio: host refused HighSpeed clock ({:?})", _e); + request.state = SdioInitState::Complete; + Ok(OperationPoll::Pending) + } + }, SdioInitState::PrepareSdSpeed => { // SAFETY: see ext_csd lend above; release happens on the // PollSdSwitchFunctionCheck Complete arm below. @@ -1597,8 +2478,15 @@ impl SdioSdmmc { Ok(CommandResponsePoll::Pending) => Ok(OperationPoll::Pending), Ok(CommandResponsePoll::Complete(_)) => { request.command_request = None; - match self.host.switch_voltage(SignalVoltage::V180) { - Ok(()) => submit_sd_access_mode_switch(self, request, mode), + match self + .host + .submit_bus_op(SdioBusOp::SwitchVoltage(SignalVoltage::V180)) + { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollSdSignalVoltage; + Ok(OperationPoll::Pending) + } Err(err) => { warn!("sdio: SD {} failed ({:?})", mode.name(), err); // SAFETY: no switch_function_request is in @@ -1621,6 +2509,24 @@ impl SdioSdmmc { } } } + SdioInitState::PollSdSignalVoltage => { + let mode = request.current_access_mode.ok_or(Error::InvalidArgument)?; + match self.poll_init_bus_op(request) { + Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), + Ok(OperationPoll::Complete(())) => { + submit_sd_access_mode_switch(self, request, mode) + } + Err(err) => { + warn!("sdio: SD {} failed ({:?})", mode.name(), err); + // SAFETY: no switch_function_request is in flight on + // this branch; the switch-status scratch slot was + // released after the earlier function-check request. + let status = + SwitchStatus::from_raw(unsafe { *request.switch_status_buf.peek() }); + submit_next_sd_access_mode(self, request, status) + } + } + } SdioInitState::PollSdSetAccessMode => { let mode = request.current_access_mode.ok_or(Error::InvalidArgument)?; let switch_request = request @@ -1639,10 +2545,57 @@ impl SdioSdmmc { warn!("sdio: SD {} failed (function mismatch)", mode.name()); submit_next_sd_access_mode(self, request, status) } else { - self.host.set_clock(mode.clock())?; - if matches!(mode, SdAccessMode::Sdr50 | SdAccessMode::Sdr104) { - self.host.execute_tuning(19)?; + match self.host.submit_bus_op(SdioBusOp::SetClock(mode.clock())) { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollSdClock; + Ok(OperationPoll::Pending) + } + Err(err) => { + warn!("sdio: SD {} failed ({:?})", mode.name(), err); + submit_next_sd_access_mode(self, request, status) + } + } + } + } + Err(err) => { + request.switch_function_request = None; + request.switch_status_buf.release(); + warn!("sdio: SD {} failed ({:?})", mode.name(), err); + // SAFETY: just released above. + let status = + SwitchStatus::from_raw(unsafe { *request.switch_status_buf.peek() }); + submit_next_sd_access_mode(self, request, status) + } + } + } + SdioInitState::PollSdClock => { + let mode = request.current_access_mode.ok_or(Error::InvalidArgument)?; + match self.poll_init_bus_op(request) { + Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), + Ok(OperationPoll::Complete(())) => { + if matches!(mode, SdAccessMode::Sdr50 | SdAccessMode::Sdr104) { + let block_size = self.sd_tuning_block_size()?; + match self.host.submit_bus_op(SdioBusOp::ExecuteTuning { + cmd_index: 19, + block_size, + }) { + Ok(bus_request) => { + request.bus_request = Some(bus_request); + request.state = SdioInitState::PollSdTuning; + Ok(OperationPoll::Pending) + } + Err(err) => { + warn!("sdio: SD {} failed ({:?})", mode.name(), err); + // SAFETY: PollSdClock is reached after the + // switch request released the status slot. + let status = SwitchStatus::from_raw(unsafe { + *request.switch_status_buf.peek() + }); + submit_next_sd_access_mode(self, request, status) + } } + } else { let status_request = self.submit_status()?; request.status_request = Some(status_request); request.state = SdioInitState::PollSdStatus; @@ -1650,10 +2603,29 @@ impl SdioSdmmc { } } Err(err) => { - request.switch_function_request = None; - request.switch_status_buf.release(); warn!("sdio: SD {} failed ({:?})", mode.name(), err); - // SAFETY: just released above. + // SAFETY: PollSdClock is reached after the switch + // request released the status slot. + let status = + SwitchStatus::from_raw(unsafe { *request.switch_status_buf.peek() }); + submit_next_sd_access_mode(self, request, status) + } + } + } + SdioInitState::PollSdTuning => { + let mode = request.current_access_mode.ok_or(Error::InvalidArgument)?; + match self.poll_init_bus_op(request) { + Ok(OperationPoll::Pending) => Ok(OperationPoll::Pending), + Ok(OperationPoll::Complete(())) => { + let status_request = self.submit_status()?; + request.status_request = Some(status_request); + request.state = SdioInitState::PollSdStatus; + Ok(OperationPoll::Pending) + } + Err(err) => { + warn!("sdio: SD {} failed ({:?})", mode.name(), err); + // SAFETY: PollSdTuning is reached after the switch + // request released the status slot. let status = SwitchStatus::from_raw(unsafe { *request.switch_status_buf.peek() }); submit_next_sd_access_mode(self, request, status) @@ -1689,9 +2661,19 @@ impl SdioSdmmc { SdioInitState::Complete => { let kind = request.kind.ok_or(Error::InvalidArgument)?; let ocr = request.ocr.ok_or(Error::InvalidArgument)?; + let ext_csd_timing = request.parsed_ext_csd.as_ref().map(|csd| csd.timing()); + let ext_csd_bus_width = request.parsed_ext_csd.as_ref().map(|csd| csd.bus_width()); info!( - "sdio: init done kind={:?} sd_v2={} high_capacity={} rca={:#x} ocr={:#x}", - kind, request.sd_v2, self.high_capacity, self.rca, ocr.raw + "sdio: init done kind={:?} sd_v2={} high_capacity={} rca={:#x} ocr={:#x} \ + host_bus_width={:?} ext_csd_bus_width={:?} ext_csd_timing={:?}", + kind, + request.sd_v2, + self.high_capacity, + self.rca, + ocr.raw, + self.bus_width, + ext_csd_bus_width, + ext_csd_timing ); Ok(OperationPoll::Complete(CardInfo { kind, @@ -1762,6 +2744,7 @@ fn submit_mmc_bus_width_or_continue<'a, H: SdioHost + 'a>( BusWidth::Bit1 => 0, BusWidth::Bit4 => 1, BusWidth::Bit8 => 2, + _ => return Err(Error::UnsupportedCommand), }; request.current_bus_width = width; request.mmc_switch_request = @@ -1770,6 +2753,24 @@ fn submit_mmc_bus_width_or_continue<'a, H: SdioHost + 'a>( Ok(OperationPoll::Pending) } +fn handle_mmc_host_bus_width_error<'a, H: SdioHost + 'a>( + driver: &mut SdioSdmmc, + request: &mut SdioInitRequest<'a, H>, + err: Error, +) -> Result, Error> { + request.bus_request = None; + if matches!(request.current_bus_width, BusWidth::Bit8) { + debug!("sdio: 8-bit refused ({:?}), trying 4-bit", err); + submit_mmc_bus_width_or_continue(driver, request, BusWidth::Bit4) + } else if matches!(request.current_bus_width, BusWidth::Bit4) { + debug!("sdio: 4-bit refused ({:?}), staying at 1-bit", err); + request.state = SdioInitState::PrepareMmcSpeed; + Ok(OperationPoll::Pending) + } else { + Err(err) + } +} + fn submit_next_sd_access_mode<'a, H: SdioHost + 'a>( driver: &mut SdioSdmmc, request: &mut SdioInitRequest<'a, H>, @@ -1854,6 +2855,7 @@ fn sd_acmd6_arg(width: BusWidth) -> Result { BusWidth::Bit1 => Ok(0), BusWidth::Bit4 => Ok(2), BusWidth::Bit8 => Err(Error::UnsupportedCommand), + _ => Err(Error::UnsupportedCommand), } } @@ -1943,9 +2945,8 @@ mod tests { /// When `Some`, `execute_tuning` returns this error. Lets the /// HS200-fallback test simulate a controller that can't tune. tuning_result: Option, - /// Records the cmd_index passed to the most recent - /// `execute_tuning` call. - last_tuning_cmd: Option, + /// Records the most recent `execute_tuning` call. + last_tuning: Option<(u8, u16)>, pending_polls: usize, /// Optional monotonic clock value returned from /// [`SdioHost::now_ms`]. Tests advance this directly to verify the @@ -1975,7 +2976,7 @@ mod tests { last_voltage: None, voltage_switch_result: None, tuning_result: None, - last_tuning_cmd: None, + last_tuning: None, pending_polls: 0, now_ms: None, } @@ -1998,7 +2999,7 @@ mod tests { last_voltage: None, voltage_switch_result: None, tuning_result: None, - last_tuning_cmd: None, + last_tuning: None, pending_polls: 0, now_ms: None, } @@ -2008,6 +3009,7 @@ mod tests { impl SdioHost for MockHost { type Event = (); type DataRequest<'a> = MockDataRequest<'a>; + type BusRequest = ReadyBusRequest; fn submit_command(&mut self, cmd: &Command) -> Result<(), Error> { self.commands.push(*cmd); @@ -2110,14 +3112,25 @@ mod tests { Ok(()) } - fn execute_tuning(&mut self, cmd_index: u8) -> Result<(), Error> { - self.last_tuning_cmd = Some(cmd_index); + fn execute_tuning(&mut self, cmd_index: u8, block_size: NonZeroU16) -> Result<(), Error> { + self.last_tuning = Some((cmd_index, block_size.get())); if let Some(e) = self.tuning_result { return Err(e); } Ok(()) } + fn submit_bus_op(&mut self, op: SdioBusOp) -> Result { + submit_ready_bus_op(self, op) + } + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, Error> { + poll_ready_bus_op(request) + } + fn now_ms(&self) -> Option { self.now_ms } @@ -2298,9 +3311,9 @@ mod tests { .host .commands .iter() - .find(|c| c.cmd == 7) + .find(|c| c.index == 7) .expect("CMD7 issued"); - assert_eq!(cmd7.arg, (0x1234u32) << 16); + assert_eq!(cmd7.argument, (0x1234u32) << 16); } #[test] @@ -2311,12 +3324,19 @@ mod tests { let mut scratch = SdioInitScratch::new(); let mut request = driver.submit_init(&mut scratch).unwrap(); + assert!(driver.host.commands.is_empty()); + for _ in 0..8 { + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + } assert_eq!( driver .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![0] ); @@ -2329,7 +3349,7 @@ mod tests { .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![0] ); @@ -2344,16 +3364,18 @@ mod tests { let mut scratch = SdioInitScratch::new(); let mut request = driver.submit_init(&mut scratch).unwrap(); - assert!(matches!( - driver.poll_init_request(&mut request).unwrap(), - OperationPoll::Pending - )); + for _ in 0..9 { + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + } assert_eq!( driver .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![0, 8] ); @@ -2367,7 +3389,7 @@ mod tests { .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![0, 8, 55] ); @@ -2394,7 +3416,7 @@ mod tests { .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![1] ); @@ -2408,16 +3430,18 @@ mod tests { .submit_init_with_preference(CardInitPreference::MmcFirst, &mut scratch) .unwrap(); - assert!(matches!( - driver.poll_init_request(&mut request).unwrap(), - OperationPoll::Pending - )); + for _ in 0..9 { + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + } assert_eq!( driver .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![0, 1] ); @@ -2439,7 +3463,7 @@ mod tests { .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![6] ); @@ -2453,7 +3477,7 @@ mod tests { .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![6, 13] ); @@ -2529,11 +3553,11 @@ mod tests { .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![13] ); - assert_eq!(driver.host.commands[0].arg, 0x1234 << 16); + assert_eq!(driver.host.commands[0].argument, 0x1234 << 16); assert!(matches!( driver.poll_status_request(&mut request).unwrap(), @@ -2555,7 +3579,7 @@ mod tests { .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![8] ); @@ -2584,7 +3608,7 @@ mod tests { .host .commands .iter() - .map(|cmd| cmd.cmd) + .map(|cmd| cmd.index) .collect::>(), std::vec![6] ); @@ -2678,9 +3702,12 @@ mod tests { assert_eq!(driver.host.last_voltage, Some(SignalVoltage::V180)); assert_eq!(driver.host.last_clock, Some(ClockSpeed::Sdr104)); - assert_eq!(driver.host.last_tuning_cmd, Some(19)); + assert_eq!( + driver.host.last_tuning, + Some((19, crate::cmd::SD_TUNING_BLOCK_SIZE as u16)) + ); assert!( - driver.host.commands.iter().any(|c| c.cmd == 11), + driver.host.commands.iter().any(|c| c.index == 11), "CMD11 issued before host voltage switch" ); assert!( @@ -2688,7 +3715,7 @@ mod tests { .host .commands .iter() - .any(|c| c.cmd == 6 && c.arg == 0x80FF_FFF3), + .any(|c| c.index == 6 && c.argument == 0x80FF_FFF3), "CMD6 switched group 1 to SDR104" ); } @@ -2721,9 +3748,9 @@ mod tests { "legacy-HighSpeed init must never ask the host for 1.8 V" ); assert_eq!(driver.host.last_clock, Some(ClockSpeed::HighSpeed)); - assert_eq!(driver.host.last_tuning_cmd, None); + assert_eq!(driver.host.last_tuning, None); assert!( - !driver.host.commands.iter().any(|c| c.cmd == 11), + !driver.host.commands.iter().any(|c| c.index == 11), "CMD11 voltage switch must not be issued in legacy HighSpeed-only mode" ); assert!( @@ -2731,7 +3758,7 @@ mod tests { .host .commands .iter() - .any(|c| c.cmd == 6 && c.arg == 0x80FF_FFF1), + .any(|c| c.index == 6 && c.argument == 0x80FF_FFF1), "CMD6 switched group 1 to HighSpeed" ); assert!( @@ -2739,7 +3766,7 @@ mod tests { .host .commands .iter() - .any(|c| c.cmd == 6 && c.arg == 0x80FF_FFF3), + .any(|c| c.index == 6 && c.argument == 0x80FF_FFF3), "SDR104 must not be selected in legacy HighSpeed-only mode" ); } @@ -2766,17 +3793,38 @@ mod tests { assert_eq!(driver.host.last_voltage, Some(SignalVoltage::V180)); assert_eq!(driver.host.last_clock, Some(ClockSpeed::HighSpeed)); - assert_eq!(driver.host.last_tuning_cmd, None); + assert_eq!(driver.host.last_tuning, None); assert!( driver .host .commands .iter() - .any(|c| c.cmd == 6 && c.arg == 0x80FF_FFF1), + .any(|c| c.index == 6 && c.argument == 0x80FF_FFF1), "CMD6 switched group 1 to HighSpeed after UHS fallback" ); } + #[test] + fn init_voltage_reset_only_ignores_unsupported() { + let mut host = MockHost::with_results(Vec::new()); + host.voltage_switch_result = Some(Error::Busy); + let mut driver = SdioSdmmc::new(host); + let mut scratch = SdioInitScratch::new(); + let mut request = driver.submit_init(&mut scratch).unwrap(); + + for _ in 0..4 { + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + } + assert!(matches!( + driver.poll_init_request(&mut request), + Err(Error::Busy) + )); + assert!(matches!(request.state, SdioInitState::ResetVoltage)); + } + #[test] fn sd_speed_selection_can_be_disabled_for_default_speed_bringup() { let replies = sd_init_replies_with_ocr(ocr_ready_sdhc_s18a()); @@ -2794,8 +3842,8 @@ mod tests { .host .commands .iter() - .filter(|c| c.cmd == 6) - .all(|c| c.arg == 2), + .filter(|c| c.index == 6) + .all(|c| c.argument == 2), "only ACMD6 bus-width switch is issued; no CMD6 SWITCH_FUNC" ); assert!( @@ -2806,7 +3854,7 @@ mod tests { .any(|e| matches!(e, MockEvent::Voltage(SignalVoltage::V180))), "speed-selection-disabled init must never ask the host for 1.8 V" ); - assert_eq!(driver.host.last_tuning_cmd, None); + assert_eq!(driver.host.last_tuning, None); } fn ocr_ready_mmc_sector() -> Response { @@ -2890,19 +3938,19 @@ mod tests { assert!(info.ext_csd.is_some()); let cmds = &driver.host.commands; - let cmd3 = cmds.iter().find(|c| c.cmd == 3).expect("CMD3 issued"); - assert_eq!(cmd3.arg, 1u32 << 16); - assert!(cmds.iter().any(|c| c.cmd == 1), "CMD1 issued"); + let cmd3 = cmds.iter().find(|c| c.index == 3).expect("CMD3 issued"); + assert_eq!(cmd3.argument, 1u32 << 16); + assert!(cmds.iter().any(|c| c.index == 1), "CMD1 issued"); // Two CMD6 SWITCHes — one for BUS_WIDTH, one for HS_TIMING. - let cmd6s: Vec<&Command> = cmds.iter().filter(|c| c.cmd == 6).collect(); + let cmd6s: Vec<&Command> = cmds.iter().filter(|c| c.index == 6).collect(); assert_eq!(cmd6s.len(), 2, "two CMD6 SWITCHes (BUS_WIDTH + HS_TIMING)"); // First: WRITE_BYTE | BUS_WIDTH(183) | value=2 (8-bit) let bw_arg = (0b11u32 << 24) | ((183u32) << 16) | (2u32 << 8); - assert_eq!(cmd6s[0].arg, bw_arg, "BUS_WIDTH=8-bit"); + assert_eq!(cmd6s[0].argument, bw_arg, "BUS_WIDTH=8-bit"); // Second: WRITE_BYTE | HS_TIMING(185) | value=1 (HS) let hs_arg = (0b11u32 << 24) | ((185u32) << 16) | (1u32 << 8); - assert_eq!(cmd6s[1].arg, hs_arg, "HS_TIMING=1"); + assert_eq!(cmd6s[1].argument, hs_arg, "HS_TIMING=1"); // Host should have ended up at 8-bit (Bit8 was accepted). assert_eq!(driver.host.bus_width, Some(BusWidth::Bit8)); @@ -3018,16 +4066,24 @@ mod tests { let _info = poll_init_to_completion(&mut driver).expect("HS200 init succeeds"); // HS_TIMING write should carry value 0x02, not 0x01. - let cmd6s: Vec<&Command> = driver.host.commands.iter().filter(|c| c.cmd == 6).collect(); + let cmd6s: Vec<&Command> = driver + .host + .commands + .iter() + .filter(|c| c.index == 6) + .collect(); // Two CMD6: BUS_WIDTH(=2) and HS_TIMING(=2) assert_eq!(cmd6s.len(), 2); let hs_timing_arg = (0b11u32 << 24) | ((185u32) << 16) | (0x02u32 << 8); - assert_eq!(cmd6s[1].arg, hs_timing_arg, "HS_TIMING=2 (HS200)"); + assert_eq!(cmd6s[1].argument, hs_timing_arg, "HS_TIMING=2 (HS200)"); // Host hooks were exercised. assert_eq!(driver.host.last_voltage, Some(SignalVoltage::V180)); assert_eq!(driver.host.last_clock, Some(ClockSpeed::Hs200)); - assert_eq!(driver.host.last_tuning_cmd, Some(21)); + assert_eq!( + driver.host.last_tuning, + Some((21, crate::cmd::MMC_TUNING_BLOCK_SIZE_8BIT as u16)) + ); let hs200_clock_pos = driver .host @@ -3043,10 +4099,10 @@ mod tests { matches!( event, MockEvent::Command(Command { - cmd: 6, - arg, + index: 6, + argument, .. - }) if *arg == hs_timing_arg + }) if *argument == hs_timing_arg ) }) .expect("HS_TIMING=2 is programmed"); @@ -3114,7 +4170,10 @@ mod tests { SignalVoltage::V330 ] ); - assert_eq!(driver.host.last_tuning_cmd, Some(21)); + assert_eq!( + driver.host.last_tuning, + Some((21, crate::cmd::MMC_TUNING_BLOCK_SIZE_8BIT as u16)) + ); // But ended up at HighSpeed, not Hs200. assert_eq!(driver.host.last_clock, Some(ClockSpeed::HighSpeed)); @@ -3124,8 +4183,8 @@ mod tests { .host .commands .iter() - .filter(|c| c.cmd == 6 && ((c.arg >> 16) & 0xFF) as u8 == 185) - .map(|c| ((c.arg >> 8) & 0xFF) as u8) + .filter(|c| c.index == 6 && ((c.argument >> 16) & 0xFF) as u8 == 185) + .map(|c| ((c.argument >> 8) & 0xFF) as u8) .collect(); assert_eq!(hs_timing_writes, std::vec![0x02, 0x01]); } @@ -3169,14 +4228,14 @@ mod tests { // because no HS200 commands were issued, but the protocol may emit // it defensively. Verify HS200 was NOT entered: no HS_TIMING=2, // no tuning, final clock is HighSpeed. - assert_eq!(driver.host.last_tuning_cmd, None); + assert_eq!(driver.host.last_tuning, None); assert_eq!(driver.host.last_clock, Some(ClockSpeed::HighSpeed)); let hs_timing_writes: Vec = driver .host .commands .iter() - .filter(|c| c.cmd == 6 && ((c.arg >> 16) & 0xFF) as u8 == 185) - .map(|c| ((c.arg >> 8) & 0xFF) as u8) + .filter(|c| c.index == 6 && ((c.argument >> 16) & 0xFF) as u8 == 185) + .map(|c| ((c.argument >> 8) & 0xFF) as u8) .collect(); assert_eq!(hs_timing_writes, std::vec![0x01]); } @@ -3212,11 +4271,11 @@ mod tests { .host .commands .iter() - .map(|c| c.cmd) + .map(|c| c.index) .collect::>(), std::vec![18] ); - assert_eq!(driver.host.commands[0].arg, 7); + assert_eq!(driver.host.commands[0].argument, 7); } #[test] @@ -3241,11 +4300,11 @@ mod tests { .host .commands .iter() - .map(|c| c.cmd) + .map(|c| c.index) .collect::>(), std::vec![25] ); - assert_eq!(driver.host.commands[0].arg, 11); + assert_eq!(driver.host.commands[0].argument, 11); assert_eq!(driver.host.writes, std::vec![buf.to_vec()]); } @@ -3319,4 +4378,360 @@ mod tests { assert_eq!(cloned.handle_irq().kind(), HostEventKind::TransferComplete); } + + struct Host2Mock { + transactions: Vec<( + Command, + Option<(sdio_host2::DataDirection, usize, u32, u32)>, + )>, + bus_ops: Vec, + response: sdio_host2::RawResponse, + transaction_error: Option, + bus_pending_polls: usize, + bus_error: Option, + transaction_aborts: usize, + bus_aborts: usize, + } + + struct Host2TransactionRequest { + response: sdio_host2::RawResponse, + pending_polls: usize, + done: bool, + } + + struct Host2BusRequest { + pending_polls: usize, + done: bool, + } + + impl sdio_host2::SdioHost for Host2Mock { + type TransactionRequest<'a> + = Host2TransactionRequest + where + Self: 'a; + type BusRequest = Host2BusRequest; + + unsafe fn submit_transaction<'a>( + &mut self, + transaction: sdio_host2::Transaction<'a>, + ) -> Result, sdio_host2::Error> + where + Self: 'a, + { + let data = transaction.data.as_ref().map(|phase| { + ( + phase.direction, + phase.buffer.len(), + u32::from(phase.block_size.get()), + phase.block_count.get(), + ) + }); + self.transactions.push((transaction.command, data)); + Ok(Host2TransactionRequest { + response: self.response, + pending_polls: 0, + done: false, + }) + } + + fn poll_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result, sdio_host2::PollRequestError> + where + Self: 'a, + { + if request.done { + return Err(sdio_host2::PollRequestError::AlreadyCompleted); + } + if request.pending_polls > 0 { + request.pending_polls -= 1; + return Ok(sdio_host2::RequestPoll::Pending); + } + if let Some(err) = self.transaction_error.take() { + request.done = true; + return Ok(sdio_host2::RequestPoll::Ready(Err(err))); + } + request.done = true; + Ok(sdio_host2::RequestPoll::Ready(Ok(request.response))) + } + + fn abort_transaction<'a>( + &mut self, + request: &mut Self::TransactionRequest<'a>, + ) -> Result<(), sdio_host2::Error> + where + Self: 'a, + { + if !request.done { + self.transaction_aborts += 1; + request.done = true; + } + Ok(()) + } + + unsafe fn submit_bus_op( + &mut self, + op: sdio_host2::BusOp, + ) -> Result { + self.bus_ops.push(op); + Ok(Host2BusRequest { + pending_polls: self.bus_pending_polls, + done: false, + }) + } + + fn poll_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result, sdio_host2::PollRequestError> { + if request.done { + return Err(sdio_host2::PollRequestError::AlreadyCompleted); + } + if request.pending_polls > 0 { + request.pending_polls -= 1; + return Ok(sdio_host2::RequestPoll::Pending); + } + if let Some(err) = self.bus_error.take() { + request.done = true; + return Ok(sdio_host2::RequestPoll::Ready(Err(err))); + } + request.done = true; + Ok(sdio_host2::RequestPoll::Ready(Ok(()))) + } + + fn abort_bus_op( + &mut self, + request: &mut Self::BusRequest, + ) -> Result<(), sdio_host2::Error> { + if !request.done { + self.bus_aborts += 1; + request.done = true; + } + Ok(()) + } + } + + impl SdioHost2Irq for Host2Mock { + type Event = (); + type IrqHandle = NoopSdioHost2IrqHandle<()>; + + fn irq_handle(&self) -> Self::IrqHandle { + NoopSdioHost2IrqHandle::default() + } + } + + impl Host2Mock { + fn new(response: sdio_host2::RawResponse) -> Self { + Self { + transactions: Vec::new(), + bus_ops: Vec::new(), + response, + transaction_error: None, + bus_pending_polls: 0, + bus_error: None, + transaction_aborts: 0, + bus_aborts: 0, + } + } + } + + #[test] + fn host2_adapter_submits_read_as_physical_transaction() { + let host = Host2Mock::new(ok_r1().to_raw_response(ResponseType::R1)); + let mut driver = SdioSdmmc::new_host2(host); + driver.high_capacity = true; + let mut buf = [0u8; 512]; + + let mut request = driver.submit_read_blocks_into(9, &mut buf).unwrap(); + assert!(matches!( + driver.poll_data_request(&mut request).unwrap(), + DataCommandPoll::Complete(Response::R1(_)) + )); + + let transactions = driver.host().with_host(|host| host.transactions.clone()); + assert_eq!(transactions.len(), 1); + assert_eq!(transactions[0].0.index, 17); + assert_eq!(transactions[0].0.argument, 9); + assert_eq!( + transactions[0].1, + Some((sdio_host2::DataDirection::Read, 512, 512, 1)) + ); + } + + #[test] + fn host2_adapter_submits_bus_ops_for_clock_changes() { + let host = Host2Mock::new(sdio_host2::RawResponse::empty()); + let mut driver = SdioSdmmc::new_host2(host); + + driver + .host_mut() + .set_clock(ClockSpeed::HighSpeed) + .expect("bus op completes"); + + assert_eq!( + driver.host().with_host(|host| host.bus_ops.clone()), + std::vec![sdio_host2::BusOp::SetClock(ClockSpeed::HighSpeed)] + ); + } + + #[test] + fn host2_adapter_poll_error_releases_active_command() { + let mut host = Host2Mock::new(ok_r1().to_raw_response(ResponseType::R1)); + host.transaction_error = Some(sdio_host2::Error::Timeout); + let mut adapter = SdioHost2Adapter::new(host); + let cmd = Command::new(13, 0, ResponseType::R1); + + adapter.submit_command(&cmd).unwrap(); + assert!(matches!( + adapter.poll_command_response(), + Err(Error::Timeout(_)) + )); + + adapter.submit_command(&cmd).unwrap(); + } + + #[test] + fn host2_sync_bus_wrapper_drains_pending_request() { + let mut host = Host2Mock::new(sdio_host2::RawResponse::empty()); + host.bus_pending_polls = 3; + let mut driver = SdioSdmmc::new_host2(host); + + driver + .host_mut() + .set_clock(ClockSpeed::HighSpeed) + .expect("compat wrapper drains pending bus request"); + + assert_eq!( + driver.host().with_host(|host| host.bus_ops.clone()), + std::vec![sdio_host2::BusOp::SetClock(ClockSpeed::HighSpeed)] + ); + } + + #[test] + fn host2_init_bus_op_pending_is_observed_without_spinning() { + let mut host = Host2Mock::new(sdio_host2::RawResponse::empty()); + host.bus_pending_polls = 1; + let mut driver = SdioSdmmc::new_host2(host); + let mut scratch = SdioInitScratch::new(); + + let mut request = driver.submit_init(&mut scratch).unwrap(); + assert!(driver.host().with_host(|host| host.bus_ops.is_empty())); + + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + assert_eq!( + driver.host().with_host(|host| host.bus_ops.clone()), + std::vec![sdio_host2::BusOp::ResetAll] + ); + assert!(driver.host().with_host(|host| host.transactions.is_empty())); + + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + assert_eq!(driver.host().with_host(|host| host.bus_ops.len()), 1); + assert!(driver.host().with_host(|host| host.transactions.is_empty())); + + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + assert_eq!(driver.host().with_host(|host| host.bus_ops.len()), 1); + assert!(driver.host().with_host(|host| host.transactions.is_empty())); + + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + assert_eq!( + driver.host().with_host(|host| host.bus_ops.clone()), + std::vec![sdio_host2::BusOp::ResetAll, sdio_host2::BusOp::PowerOn] + ); + assert!(driver.host().with_host(|host| host.transactions.is_empty())); + } + + #[test] + fn host2_init_starts_with_physical_bus_ops_before_cmd0() { + let host = Host2Mock::new(sdio_host2::RawResponse::empty()); + let mut driver = SdioSdmmc::new_host2(host); + let mut scratch = SdioInitScratch::new(); + let mut request = driver.submit_init(&mut scratch).unwrap(); + + for _ in 0..16 { + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + if driver + .host() + .with_host(|host| !host.transactions.is_empty()) + { + break; + } + } + + assert_eq!( + driver.host().with_host(|host| host.bus_ops.clone()), + std::vec![ + sdio_host2::BusOp::ResetAll, + sdio_host2::BusOp::PowerOn, + sdio_host2::BusOp::SetSignalVoltage(SignalVoltage::V330), + sdio_host2::BusOp::SetBusWidth(BusWidth::Bit1), + sdio_host2::BusOp::SetClock(ClockSpeed::Identification), + ] + ); + let transactions = driver.host().with_host(|host| host.transactions.clone()); + assert_eq!(transactions.len(), 1); + assert_eq!(transactions[0].0.index, 0); + assert!(transactions[0].1.is_none()); + } + + #[test] + fn host2_init_bus_op_error_releases_request_slot() { + let mut host = Host2Mock::new(sdio_host2::RawResponse::empty()); + host.bus_error = Some(sdio_host2::Error::Timeout); + let mut driver = SdioSdmmc::new_host2(host); + let mut scratch = SdioInitScratch::new(); + let mut request = driver.submit_init(&mut scratch).unwrap(); + + assert!(matches!( + driver.poll_init_request(&mut request).unwrap(), + OperationPoll::Pending + )); + assert!(matches!( + driver.poll_init_request(&mut request), + Err(Error::Timeout(_)) + )); + assert!(request.bus_request.is_none()); + } + + #[test] + fn host2_adapter_drop_aborts_pending_data_request() { + let host = Host2Mock::new(ok_r1().to_raw_response(ResponseType::R1)); + let mut adapter = SdioHost2Adapter::new(host); + let cmd = Command::new(17, 0, ResponseType::R1); + let mut buf = [0u8; 512]; + + let request = adapter.submit_read_data(&cmd, &mut buf, 512, 1).unwrap(); + drop(request); + + assert_eq!(adapter.with_host(|host| host.transaction_aborts), 1); + } + + #[test] + fn host2_sync_bus_timeout_aborts_pending_bus_request() { + let mut host = Host2Mock::new(sdio_host2::RawResponse::empty()); + host.bus_pending_polls = (SDIO_HOST2_COMPAT_POLL_LIMIT as usize) + 1; + let mut adapter = SdioHost2Adapter::new(host); + + assert!(matches!( + adapter.set_clock(ClockSpeed::HighSpeed), + Err(Error::Timeout(_)) + )); + + assert_eq!(adapter.with_host(|host| host.bus_aborts), 1); + } } diff --git a/drivers/blk/sdmmc-protocol/src/spi.rs b/drivers/blk/sdmmc-protocol/src/spi.rs index ddf2785b75..f2cf100751 100644 --- a/drivers/blk/sdmmc-protocol/src/spi.rs +++ b/drivers/blk/sdmmc-protocol/src/spi.rs @@ -298,7 +298,7 @@ impl SpiSdmmc { Response::R1(r1) | Response::R1b(r1) => Ok(r1), _ => Err(Error::BadResponse(ErrorContext::for_cmd( Phase::ResponseWait, - cmd.cmd, + cmd.index, ))), } } @@ -310,7 +310,7 @@ impl SpiSdmmc { self.transport.send_byte(b)?; } - let response = match cmd.resp_type { + let response = match cmd.response { ResponseType::None => { let r1 = self.read_r1()?; Ok(Response::R1(r1)) diff --git a/drivers/interface/rdif-block/Cargo.toml b/drivers/interface/rdif-block/Cargo.toml index fb0ee49a4d..2f626b4a58 100644 --- a/drivers/interface/rdif-block/Cargo.toml +++ b/drivers/interface/rdif-block/Cargo.toml @@ -12,4 +12,5 @@ version = "0.9.1" [dependencies] dma-api = {workspace = true} rdif-base = {workspace = true} +spin = {workspace = true} thiserror = {version = "2", default-features = false} diff --git a/drivers/interface/rdif-block/src/info.rs b/drivers/interface/rdif-block/src/info.rs index 762a67d59e..c7eff4dbf8 100644 --- a/drivers/interface/rdif-block/src/info.rs +++ b/drivers/interface/rdif-block/src/info.rs @@ -1,3 +1,5 @@ +use dma_api::DmaDomainId; + use crate::request::RequestFlags; #[derive(Debug, Clone, Copy)] @@ -26,6 +28,7 @@ impl DeviceInfo { #[derive(Debug, Clone, Copy)] pub struct QueueLimits { pub dma_mask: u64, + pub dma_domain: DmaDomainId, pub dma_alignment: usize, pub max_inflight: usize, pub max_blocks_per_request: u32, @@ -41,6 +44,7 @@ impl QueueLimits { pub const fn simple(logical_block_size: usize, dma_mask: u64) -> Self { Self { dma_mask, + dma_domain: DmaDomainId::legacy_global(), dma_alignment: logical_block_size, max_inflight: 1, max_blocks_per_request: 1, diff --git a/drivers/interface/rdif-block/src/interface.rs b/drivers/interface/rdif-block/src/interface.rs index db0efaad62..b4ea1f5951 100644 --- a/drivers/interface/rdif-block/src/interface.rs +++ b/drivers/interface/rdif-block/src/interface.rs @@ -1,16 +1,27 @@ -use alloc::{boxed::Box, vec::Vec}; +use alloc::{boxed::Box, sync::Arc, vec::Vec}; + +use spin::Mutex; use crate::{ - BlkError, DeviceInfo, DriverGeneric, IrqHandler, IrqSourceList, QueueInfo, QueueLimits, - Request, RequestId, RequestStatus, + BlkError, DeviceInfo, DriverGeneric, IrqHandler, IrqSourceList, OwnedRequest, PollError, + QueueInfo, QueueLimits, Request, RequestId, RequestPoll, RequestStatus, SubmitError, }; +pub type BInterface = Box; +pub type BQueue = Box; +pub type BOwnedQueue = Box; +pub type BIrqHandler = Box; + pub trait Interface: DriverGeneric { fn device_info(&self) -> DeviceInfo; fn queue_limits(&self) -> QueueLimits; - fn create_queue(&mut self) -> Option>; + fn create_queue(&mut self) -> Option; + + fn create_owned_queue(&mut self) -> Option { + None + } fn enable_irq(&self) {} @@ -24,7 +35,7 @@ pub trait Interface: DriverGeneric { Vec::new() } - fn take_irq_handler(&mut self, _source_id: usize) -> Option> { + fn take_irq_handler(&mut self, _source_id: usize) -> Option { None } } @@ -33,6 +44,125 @@ pub trait CompletionSink { fn complete(&mut self, request: RequestId, result: Result<(), BlkError>); } +/// A request queue that owns DMA backing for every in-flight request. +pub trait IQueueOwned: Send + 'static { + fn id(&self) -> usize; + + fn info(&self) -> QueueInfo; + + fn submit_request(&mut self, request: OwnedRequest) -> Result; + + fn poll_request(&mut self, request: RequestId) -> Result; + + fn cancel_request(&mut self, request: RequestId) -> Result; + + fn shutdown(&mut self) {} +} + +pub struct QueueHandle { + queue: Option, +} + +impl QueueHandle { + pub fn new(queue: BOwnedQueue) -> Self { + Self { queue: Some(queue) } + } + + pub fn id(&self) -> usize { + self.queue + .as_ref() + .expect("owned queue handle must contain queue") + .id() + } + + pub fn info(&self) -> QueueInfo { + self.queue + .as_ref() + .expect("owned queue handle must contain queue") + .info() + } + + pub fn submit_request(&mut self, request: OwnedRequest) -> Result { + self.queue + .as_mut() + .expect("owned queue handle must contain queue") + .submit_request(request) + } + + pub fn poll_request(&mut self, request: RequestId) -> Result { + self.queue + .as_mut() + .expect("owned queue handle must contain queue") + .poll_request(request) + } + + pub fn cancel_request(&mut self, request: RequestId) -> Result { + self.queue + .as_mut() + .expect("owned queue handle must contain queue") + .cancel_request(request) + } + + pub fn shutdown(&mut self) { + if let Some(queue) = self.queue.as_mut() { + queue.shutdown(); + } + } +} + +impl Drop for QueueHandle { + fn drop(&mut self) { + self.shutdown(); + } +} + +#[derive(Clone)] +pub struct IrqHandlerSlot { + inner: Arc>>, +} + +impl IrqHandlerSlot { + pub fn new(handler: BIrqHandler) -> Self { + Self { + inner: Arc::new(Mutex::new(Some(handler))), + } + } + + pub fn take(&self) -> Option { + let handler = self.inner.lock().take()?; + Some(IrqHandlerHandle { + slot: Self { + inner: Arc::clone(&self.inner), + }, + handler: Some(handler), + }) + } +} + +pub struct IrqHandlerHandle { + slot: IrqHandlerSlot, + handler: Option, +} + +impl IrqHandler for IrqHandlerHandle { + fn handle_irq(&self) -> crate::Event { + self.handler + .as_ref() + .expect("IRQ handler handle must contain handler") + .handle_irq() + } +} + +impl Drop for IrqHandlerHandle { + fn drop(&mut self) { + if let Some(handler) = self.handler.take() { + let mut slot = self.slot.inner.lock(); + debug_assert!(slot.is_none()); + *slot = Some(handler); + } + } +} + /// A request queue for one block device hardware/software queue. /// /// # Safety @@ -199,4 +329,61 @@ mod tests { assert_eq!(limits.max_inflight, 1); } + + #[test] + fn irq_handler_slot_returns_handler_on_drop() { + let slot = IrqHandlerSlot::new(Box::new(NoopIrq)); + let handler = slot.take().unwrap(); + + assert!(slot.take().is_none()); + assert!(handler.handle_irq().queues.contains(1)); + + drop(handler); + assert!(slot.take().is_some()); + } + + struct OwnedQueue; + + impl IQueueOwned for OwnedQueue { + fn id(&self) -> usize { + 2 + } + + fn info(&self) -> QueueInfo { + QueueInfo { + id: 2, + device: DeviceInfo::new(8, 512), + limits: QueueLimits::simple(512, u64::MAX), + } + } + + fn submit_request(&mut self, request: OwnedRequest) -> Result { + Err(SubmitError::new(BlkError::Retry, request)) + } + + fn poll_request(&mut self, _request: RequestId) -> Result { + Ok(RequestPoll::Pending) + } + + fn cancel_request(&mut self, _request: RequestId) -> Result { + Err(PollError::UnknownRequest) + } + } + + #[test] + fn owned_queue_submit_error_returns_request_ownership() { + let mut handle = QueueHandle::new(Box::new(OwnedQueue)); + let request = OwnedRequest { + op: RequestOp::Flush, + lba: 0, + block_count: 0, + data: None, + flags: Default::default(), + }; + + let err = handle.submit_request(request).unwrap_err(); + + assert_eq!(err.error, BlkError::Retry); + assert!(matches!(err.request().op, RequestOp::Flush)); + } } diff --git a/drivers/interface/rdif-block/src/lib.rs b/drivers/interface/rdif-block/src/lib.rs index 6be9f6f95e..687dbbe785 100644 --- a/drivers/interface/rdif-block/src/lib.rs +++ b/drivers/interface/rdif-block/src/lib.rs @@ -12,7 +12,10 @@ mod request; pub use dma_api; pub use error::BlkError; pub use info::{DeviceInfo, QueueInfo, QueueLimits}; -pub use interface::{CompletionSink, IQueue, Interface}; +pub use interface::{ + BInterface, BIrqHandler, BOwnedQueue, BQueue, CompletionSink, IQueue, IQueueOwned, Interface, + IrqHandlerHandle, IrqHandlerSlot, QueueHandle, +}; pub use irq::{ CompletionHint, CompletionIds, CompletionList, Event, IdList, IrqHandler, IrqSourceInfo, IrqSourceList, MAX_BATCH_COMPLETION_IDS, MAX_COMPLETION_HINTS, @@ -23,6 +26,7 @@ pub use planner::{ }; pub use rdif_base::{DriverGeneric, KError, io}; pub use request::{ - Buffer, Request, RequestFlags, RequestId, RequestOp, RequestStatus, Segment, validate_request, - validate_request_shape, + Buffer, CompletedRequest, OwnedRequest, PollError, Request, RequestFlags, RequestId, RequestOp, + RequestPoll, RequestStatus, Segment, SubmitError, validate_owned_request, + validate_owned_request_shape, validate_request, validate_request_shape, }; diff --git a/drivers/interface/rdif-block/src/planner.rs b/drivers/interface/rdif-block/src/planner.rs index 523e8ce68c..9c5a90e6d7 100644 --- a/drivers/interface/rdif-block/src/planner.rs +++ b/drivers/interface/rdif-block/src/planner.rs @@ -256,6 +256,7 @@ mod tests { ) -> QueueLimits { QueueLimits { dma_mask: u64::MAX, + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_alignment: 512, max_inflight: 1, max_blocks_per_request, diff --git a/drivers/interface/rdif-block/src/request.rs b/drivers/interface/rdif-block/src/request.rs index 6f2950764d..f7a16a7ae2 100644 --- a/drivers/interface/rdif-block/src/request.rs +++ b/drivers/interface/rdif-block/src/request.rs @@ -1,8 +1,11 @@ +use alloc::boxed::Box; use core::{ marker::PhantomData, ops::{Deref, DerefMut}, }; +use dma_api::{CompletedDma, PreparedDma}; + use crate::{BlkError, DeviceInfo, QueueInfo, QueueLimits}; #[repr(transparent)] @@ -158,8 +161,93 @@ impl Request<'_> { } } +/// Block I/O request that moves DMA backing ownership into the queue. +pub struct OwnedRequest { + pub op: RequestOp, + pub lba: u64, + pub block_count: u32, + pub data: Option, + pub flags: RequestFlags, +} + +impl OwnedRequest { + pub fn data_len(&self) -> usize { + self.data.as_ref().map_or(0, |data| data.len().get()) + } + + pub fn is_data_op(&self) -> bool { + matches!(self.op, RequestOp::Read | RequestOp::Write) + } +} + +/// Submit-side failure that returns request ownership to the caller. +pub struct SubmitError { + pub error: BlkError, + request: Box, +} + +impl SubmitError { + pub fn new(error: BlkError, request: OwnedRequest) -> Self { + Self { + error, + request: Box::new(request), + } + } + + pub fn into_request(self) -> OwnedRequest { + *self.request + } + + pub fn request(&self) -> &OwnedRequest { + &self.request + } +} + +/// One terminal request result with owned DMA backing returned to the runtime. +pub struct CompletedRequest { + pub id: RequestId, + pub result: Result<(), BlkError>, + pub data: Option, +} + +impl CompletedRequest { + pub const fn new( + id: RequestId, + result: Result<(), BlkError>, + data: Option, + ) -> Self { + Self { id, result, data } + } +} + +/// Result of polling an owned queue request. +pub enum RequestPoll { + Pending, + Ready(CompletedRequest), +} + +/// Queue misuse/query failure. +/// +/// This is intentionally separate from request I/O completion. Returning this +/// error must not tell the runtime that the DMA backing is safe to recycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PollError { + UnknownRequest, + WrongQueue, + DriverPoisoned, +} + +impl From for BlkError { + fn from(value: PollError) -> Self { + match value { + PollError::UnknownRequest | PollError::WrongQueue => BlkError::InvalidRequest, + PollError::DriverPoisoned => BlkError::Io, + } + } +} + pub fn validate_request(info: QueueInfo, request: &Request<'_>) -> Result<(), BlkError> { - validate_request_flags(info, request)?; + validate_request_flags(info, request.flags)?; validate_request_shape(info.device, info.limits, request) } @@ -235,18 +323,97 @@ pub fn validate_request_shape( Ok(()) } -fn validate_request_flags(info: QueueInfo, request: &Request<'_>) -> Result<(), BlkError> { - let unknown = request.flags.unsupported_by(RequestFlags::ALL_KNOWN); +pub fn validate_owned_request(info: QueueInfo, request: &OwnedRequest) -> Result<(), BlkError> { + validate_request_flags(info, request.flags)?; + validate_owned_request_shape(info.device, info.limits, request) +} + +pub fn validate_owned_request_shape( + info: DeviceInfo, + limits: QueueLimits, + request: &OwnedRequest, +) -> Result<(), BlkError> { + if request.block_count == 0 && !matches!(request.op, RequestOp::Flush) { + return Err(BlkError::InvalidRequest); + } + + if request.lba >= info.num_blocks + || request + .lba + .checked_add(request.block_count as u64) + .is_none_or(|end| end > info.num_blocks) + { + return Err(BlkError::InvalidBlockIndex(request.lba)); + } + + match request.op { + RequestOp::Read | RequestOp::Write => { + let expected = request + .block_count + .checked_mul(info.logical_block_size as u32) + .map(|len| len as usize) + .ok_or(BlkError::InvalidRequest)?; + if request.data_len() != expected { + return Err(BlkError::InvalidRequest); + } + let Some(data) = &request.data else { + return Err(BlkError::InvalidRequest); + }; + let segments = data.segments(); + if segments.is_empty() + || segments.len() > limits.max_segments + || segments + .iter() + .any(|segment| segment.len.get() > limits.max_segment_size) + { + return Err(BlkError::InvalidRequest); + } + } + RequestOp::Flush => { + if request.data.is_some() || request.block_count != 0 { + return Err(BlkError::InvalidRequest); + } + if !limits.supports_flush { + return Err(BlkError::NotSupported); + } + } + RequestOp::Discard => { + if request.data.is_some() { + return Err(BlkError::InvalidRequest); + } + if !limits.supports_discard { + return Err(BlkError::NotSupported); + } + } + RequestOp::WriteZeroes => { + if request.data.is_some() { + return Err(BlkError::InvalidRequest); + } + if !limits.supports_write_zeroes { + return Err(BlkError::NotSupported); + } + } + } + + if request.block_count > limits.max_blocks_per_request { + return Err(BlkError::InvalidRequest); + } + + Ok(()) +} + +fn validate_request_flags(info: QueueInfo, flags: RequestFlags) -> Result<(), BlkError> { + let unknown = flags.unsupported_by(RequestFlags::ALL_KNOWN); if !unknown.is_empty() { return Err(BlkError::InvalidRequest); } - let unsupported = request.flags.unsupported_by(info.limits.supported_flags); + let unsupported = flags.unsupported_by(info.limits.supported_flags); if !unsupported.is_empty() { return Err(BlkError::NotSupported); } - if request.flags.intersects(RequestFlags::PREFLUSH) && !info.limits.supports_flush { + if flags.intersects(RequestFlags::PREFLUSH) && !info.limits.supports_flush { return Err(BlkError::NotSupported); } @@ -408,6 +575,7 @@ mod tests { fn request_validation_rejects_transfer_larger_than_hard_block_limit() { let info = queue_info_with(QueueLimits { dma_mask: u64::MAX, + dma_domain: dma_api::DmaDomainId::legacy_global(), dma_alignment: 512, max_inflight: 1, max_blocks_per_request: 2, diff --git a/drivers/net/eth-intel/src/e1000/mod.rs b/drivers/net/eth-intel/src/e1000/mod.rs index 7510b52f32..b4c673c9fe 100644 --- a/drivers/net/eth-intel/src/e1000/mod.rs +++ b/drivers/net/eth-intel/src/e1000/mod.rs @@ -44,7 +44,7 @@ impl E1000 { mmio_api::init(mmio_op); let mmio = mmio_api::ioremap(bar_addr.into(), bar_size)?; let regs = Regs::new(mmio.as_nonnull_ptr()); - let dma = DeviceDma::new(dma_mask, dma_op); + let dma = DeviceDma::new_legacy(dma_mask, dma_op); regs.reset(); regs.disable_all_irq(); diff --git a/drivers/net/rd-net/src/lib.rs b/drivers/net/rd-net/src/lib.rs index 26a083e659..590c4a5545 100644 --- a/drivers/net/rd-net/src/lib.rs +++ b/drivers/net/rd-net/src/lib.rs @@ -242,7 +242,7 @@ fn make_pool( ) -> Result { let layout = Layout::from_size_align(config.buf_size, config.align.max(1)) .map_err(|_| other_error("invalid queue layout"))?; - let dma = DeviceDma::new(config.dma_mask, dma_op); + let dma = DeviceDma::new_legacy(config.dma_mask, dma_op); Ok(dma.contiguous_buffer_pool(layout, direction, config.ring_size)) } diff --git a/drivers/net/realtek-rtl8125/src/lib.rs b/drivers/net/realtek-rtl8125/src/lib.rs index b871cfbe09..4a0d4f07ad 100644 --- a/drivers/net/realtek-rtl8125/src/lib.rs +++ b/drivers/net/realtek-rtl8125/src/lib.rs @@ -112,7 +112,7 @@ impl Rtl8125 { mmio_api::init(mmio_op); let mmio = mmio_api::ioremap(bar_addr.into(), bar_size.max(RTL8125_REGS_SIZE))?; let regs = Regs::new(mmio.as_nonnull_ptr()); - let dma = DeviceDma::new(dma_mask, dma_op); + let dma = DeviceDma::new_legacy(dma_mask, dma_op); let xid = rtl8125_xid(regs); let chip = chip_version(xid); diff --git a/drivers/npu/rockchip-npu/tests/test.rs b/drivers/npu/rockchip-npu/tests/test.rs index 1b649414ef..81824bbd51 100644 --- a/drivers/npu/rockchip-npu/tests/test.rs +++ b/drivers/npu/rockchip-npu/tests/test.rs @@ -44,7 +44,7 @@ mod tests { // static IRQ_STATUS: AtomicU32 = AtomicU32::new(0); // fn dma_device() -> DeviceDma { - // DeviceDma::new(u32::MAX as u64, kernel_dma_op()) + // DeviceDma::new_legacy(u32::MAX as u64, kernel_dma_op()) // } // #[test] diff --git a/drivers/usb/usb-host/src/backend/kmod/osal.rs b/drivers/usb/usb-host/src/backend/kmod/osal.rs index 6b4f13d566..3b32b9f9e1 100644 --- a/drivers/usb/usb-host/src/backend/kmod/osal.rs +++ b/drivers/usb/usb-host/src/backend/kmod/osal.rs @@ -12,7 +12,7 @@ pub(crate) struct Kernel { impl Kernel { pub fn new(dma_mask: u64, osal: &'static dyn KernelOp) -> Self { Self { - dma: DeviceDma::new(dma_mask, osal), + dma: DeviceDma::new_legacy(dma_mask, osal), osal, } } diff --git a/memory/dma-api/README.md b/memory/dma-api/README.md index c394c71ab5..af4a5d043b 100644 --- a/memory/dma-api/README.md +++ b/memory/dma-api/README.md @@ -42,12 +42,12 @@ pub struct DmaConstraints { } ``` -`DeviceDma::new(dma_mask, op)` is shorthand for +`DeviceDma::new_legacy(dma_mask, op)` is shorthand for `DmaConstraints::new(dma_mask)`. Use `with_constraints` when a specific queue or transfer has stronger alignment, boundary, or segment-size requirements. Backends must never hand a driver a DMA address outside the requested mask. For -example, a device created with `DeviceDma::new(u32::MAX as u64, op)` must only +example, a device created with `DeviceDma::new_legacy(u32::MAX as u64, op)` must only return 32-bit reachable DMA addresses. Streaming mappings may use a fast path when the original buffer already satisfies the constraints; otherwise they should allocate an in-mask bounce buffer. diff --git a/memory/dma-api/src/array.rs b/memory/dma-api/src/array.rs index 650c331fdd..57d6399619 100644 --- a/memory/dma-api/src/array.rs +++ b/memory/dma-api/src/array.rs @@ -1,6 +1,9 @@ use core::{alloc::Layout, marker::PhantomData, ptr::NonNull}; -use crate::{DeviceDma, DmaAddr, DmaDirection, DmaError, DmaPod, common::DmaAllocation}; +use crate::{ + DeviceDma, DmaAddr, DmaDirection, DmaDomainId, DmaError, DmaPod, + common::{AllocationKind, DmaAllocation}, +}; pub struct CoherentArray { data: DmaAllocation, @@ -138,6 +141,17 @@ impl ContiguousArray { self.data.handle.size() } + pub fn domain_id(&self) -> DmaDomainId { + self.data.device.domain_id() + } + + pub fn direction(&self) -> DmaDirection { + match self.data.kind { + AllocationKind::Contiguous { direction } => direction, + AllocationKind::Coherent => unreachable!("ContiguousArray cannot hold coherent DMA"), + } + } + pub fn read_cpu(&self, index: usize) -> Option { read_at(self.as_ptr(), self.len(), index) } diff --git a/memory/dma-api/src/common.rs b/memory/dma-api/src/common.rs index 8e7b771a70..7e4c61c237 100644 --- a/memory/dma-api/src/common.rs +++ b/memory/dma-api/src/common.rs @@ -2,6 +2,7 @@ use core::alloc::Layout; use crate::{DeviceDma, DmaAllocHandle, DmaDirection, DmaError}; +#[derive(Clone, Copy)] pub(crate) enum AllocationKind { Coherent, Contiguous { direction: DmaDirection }, diff --git a/memory/dma-api/src/def.rs b/memory/dma-api/src/def.rs index fe88785070..8e26238659 100644 --- a/memory/dma-api/src/def.rs +++ b/memory/dma-api/src/def.rs @@ -1,4 +1,4 @@ -use core::{alloc::Layout, cmp::PartialOrd, ptr::NonNull}; +use core::{alloc::Layout, cmp::PartialOrd, num::NonZeroU64, ptr::NonNull}; use derive_more::{ Add, AddAssign, Debug, Display, Div, From, Into, Mul, MulAssign, Sub, SubAssign, @@ -49,6 +49,33 @@ impl PartialOrd for DmaAddr { } } +/// Stable identity for one DMA translation domain. +/// +/// Drivers use this to reject already-prepared DMA buffers that were prepared +/// for a different device/IOMMU domain. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DmaDomainId(NonZeroU64); + +impl DmaDomainId { + pub const fn new(id: NonZeroU64) -> Self { + Self(id) + } + + /// Compatibility domain for legacy callers that have not plumbed a + /// device/IOMMU-specific identity yet. + pub const fn legacy_global() -> Self { + Self(NonZeroU64::MIN) + } + + pub fn from_raw(id: u64) -> Self { + Self(NonZeroU64::new(id).unwrap_or(NonZeroU64::MIN)) + } + + pub const fn get(self) -> NonZeroU64 { + self.0 + } +} + /// Device-visible DMA constraints. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DmaConstraints { diff --git a/memory/dma-api/src/lib.rs b/memory/dma-api/src/lib.rs index 01797f3055..f0c5242adf 100644 --- a/memory/dma-api/src/lib.rs +++ b/memory/dma-api/src/lib.rs @@ -11,6 +11,7 @@ mod array; mod common; mod dbox; mod def; +mod owned; mod pool; mod streaming; @@ -18,6 +19,7 @@ pub use array::*; pub use dbox::*; pub use def::*; pub use op::DmaOp; +pub use owned::*; pub use pool::*; pub use streaming::*; @@ -25,20 +27,27 @@ pub use streaming::*; pub struct DeviceDma { op: &'static dyn DmaOp, constraints: DmaConstraints, + domain: DmaDomainId, } impl DeviceDma { - pub fn new(dma_mask: u64, op: &'static dyn DmaOp) -> Self { + pub fn new(domain: DmaDomainId, dma_mask: u64, op: &'static dyn DmaOp) -> Self { Self { constraints: DmaConstraints::new(dma_mask), + domain, op, } } + pub fn new_legacy(dma_mask: u64, op: &'static dyn DmaOp) -> Self { + Self::new(DmaDomainId::legacy_global(), dma_mask, op) + } + pub fn with_constraints(&self, constraints: DmaConstraints) -> Self { Self { op: self.op, constraints, + domain: self.domain, } } @@ -50,6 +59,10 @@ impl DeviceDma { self.constraints.addr_mask } + pub fn domain_id(&self) -> DmaDomainId { + self.domain + } + pub fn flush(&self, addr: NonNull, size: usize) { self.op.flush(addr, size) } diff --git a/memory/dma-api/src/owned.rs b/memory/dma-api/src/owned.rs new file mode 100644 index 0000000000..dcb03fec97 --- /dev/null +++ b/memory/dma-api/src/owned.rs @@ -0,0 +1,261 @@ +use core::{mem::ManuallyDrop, num::NonZeroUsize, ptr::NonNull}; + +use crate::{ContiguousArray, DeviceDma, DmaAddr, DmaDirection, DmaDomainId, DmaError}; + +/// One device-visible DMA segment owned by a prepared request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DmaSegment { + pub addr: DmaAddr, + pub len: NonZeroUsize, +} + +impl DmaSegment { + pub const fn new(addr: DmaAddr, len: NonZeroUsize) -> Self { + Self { addr, len } + } +} + +/// CPU-owned contiguous DMA buffer that can be prepared for one async request. +pub struct CpuDmaBuffer { + backing: ContiguousArray, + direction: DmaDirection, + domain: DmaDomainId, +} + +impl CpuDmaBuffer { + pub fn new_zero( + device: &DeviceDma, + len: NonZeroUsize, + align: usize, + direction: DmaDirection, + ) -> Result { + let backing = + device.contiguous_array_zero_with_align(len.get(), align.max(1), direction)?; + Ok(Self::from_contiguous(backing)) + } + + pub fn from_contiguous(backing: ContiguousArray) -> Self { + assert!( + !backing.is_empty(), + "CpuDmaBuffer backing must be non-empty" + ); + let direction = backing.direction(); + let domain = backing.domain_id(); + Self { + backing, + direction, + domain, + } + } + + pub fn len(&self) -> NonZeroUsize { + NonZeroUsize::new(self.backing.bytes_len()) + .expect("CpuDmaBuffer never owns zero-sized backing") + } + + pub fn is_empty(&self) -> bool { + false + } + + pub const fn direction(&self) -> DmaDirection { + self.direction + } + + pub const fn domain_id(&self) -> DmaDomainId { + self.domain + } + + pub fn cpu_ptr(&self) -> NonNull { + self.backing.as_ptr() + } + + pub fn dma_addr(&self) -> DmaAddr { + self.backing.dma_addr() + } + + pub fn segment(&self) -> DmaSegment { + DmaSegment::new(self.dma_addr(), self.len()) + } + + pub fn as_slice_cpu(&self) -> &[u8] { + self.backing.as_slice_cpu() + } + + /// # Safety + /// + /// The caller must ensure no device can access this buffer while the + /// returned mutable CPU slice is used. + pub unsafe fn as_mut_slice_cpu(&mut self) -> &mut [u8] { + unsafe { self.backing.as_mut_slice_cpu() } + } + + pub fn copy_to_device_from_slice(&mut self, src: &[u8]) { + self.backing.copy_to_device_from_slice(src); + } + + pub fn copy_from_device_to_slice(&self, dst: &mut [u8]) { + self.backing.copy_from_device_to_slice(dst); + } + + pub fn prepare_for_device_all(&self) { + self.backing.prepare_for_device_all(); + } + + pub fn complete_for_cpu_all(&self) { + self.backing.complete_for_cpu_all(); + } + + pub fn prepare_for_device(self) -> PreparedDma { + self.prepare_for_device_all(); + PreparedDma { buffer: self } + } +} + +/// DMA backing prepared for device access but not yet owned by hardware. +pub struct PreparedDma { + buffer: CpuDmaBuffer, +} + +impl PreparedDma { + pub fn len(&self) -> NonZeroUsize { + self.buffer.len() + } + + pub const fn direction(&self) -> DmaDirection { + self.buffer.direction() + } + + pub const fn domain_id(&self) -> DmaDomainId { + self.buffer.domain_id() + } + + pub fn cpu_ptr(&self) -> NonNull { + self.buffer.cpu_ptr() + } + + pub fn dma_addr(&self) -> DmaAddr { + self.buffer.dma_addr() + } + + pub fn segment(&self) -> DmaSegment { + self.buffer.segment() + } + + pub fn segments(&self) -> [DmaSegment; 1] { + [self.segment()] + } + + pub fn into_cpu_buffer(self) -> CpuDmaBuffer { + self.buffer + } + + /// # Safety + /// + /// The caller must start hardware ownership using this prepared backing + /// and later return it only after hardware is quiesced. + pub unsafe fn into_in_flight(self) -> InFlightDma { + InFlightDma { + prepared: ManuallyDrop::new(self), + } + } +} + +/// DMA backing currently owned by a hardware request. +/// +/// Dropping this object intentionally leaks the backing as a last-resort +/// quarantine: safe callers must not observe memory reuse while hardware could +/// still be accessing it. +pub struct InFlightDma { + prepared: ManuallyDrop, +} + +impl InFlightDma { + pub fn len(&self) -> NonZeroUsize { + self.prepared.len() + } + + pub fn direction(&self) -> DmaDirection { + self.prepared.direction() + } + + pub fn domain_id(&self) -> DmaDomainId { + self.prepared.domain_id() + } + + pub fn cpu_ptr(&self) -> NonNull { + self.prepared.cpu_ptr() + } + + pub fn dma_addr(&self) -> DmaAddr { + self.prepared.dma_addr() + } + + pub fn segment(&self) -> DmaSegment { + self.prepared.segment() + } + + /// # Safety + /// + /// The caller must have stopped DMA bus-master access and any command/data + /// engine that can touch this exact in-flight backing. + pub unsafe fn complete_after_quiesce(mut self) -> CompletedDma { + let prepared = unsafe { ManuallyDrop::take(&mut self.prepared) }; + if matches!( + prepared.buffer.direction(), + DmaDirection::FromDevice | DmaDirection::Bidirectional + ) { + prepared.buffer.complete_for_cpu_all(); + } + CompletedDma { + buffer: prepared.buffer, + } + } + + pub fn quarantine(mut self) -> QuarantinedDma { + let prepared = unsafe { ManuallyDrop::take(&mut self.prepared) }; + QuarantinedDma { + prepared: ManuallyDrop::new(prepared), + } + } +} + +/// DMA backing completed by hardware and visible to CPU again. +pub struct CompletedDma { + buffer: CpuDmaBuffer, +} + +impl CompletedDma { + pub fn len(&self) -> NonZeroUsize { + self.buffer.len() + } + + pub const fn direction(&self) -> DmaDirection { + self.buffer.direction() + } + + pub fn copy_from_device_to_slice(&self, dst: &mut [u8]) { + self.buffer.copy_from_device_to_slice(dst); + } + + pub fn into_cpu_buffer(self) -> CpuDmaBuffer { + self.buffer + } +} + +/// DMA backing that cannot yet be safely recycled. +/// +/// This type deliberately has no accessor to recover the CPU buffer. Dropping +/// it leaks the backing, preserving the safety invariant. +pub struct QuarantinedDma { + prepared: ManuallyDrop, +} + +impl QuarantinedDma { + pub fn len(&self) -> NonZeroUsize { + self.prepared.len() + } + + pub fn domain_id(&self) -> DmaDomainId { + self.prepared.domain_id() + } +} diff --git a/memory/dma-api/tests/test.rs b/memory/dma-api/tests/test.rs index 55e25e0b02..d39b2f474e 100644 --- a/memory/dma-api/tests/test.rs +++ b/memory/dma-api/tests/test.rs @@ -16,7 +16,7 @@ struct Descriptor { fn new_tracking_device() -> (DeviceDma, &'static TrackingDmaOp) { let tracker = Box::new(TrackingDmaOp::new()); let tracker = Box::leak(tracker); - (DeviceDma::new(u64::MAX, tracker), tracker) + (DeviceDma::new_legacy(u64::MAX, tracker), tracker) } #[test] @@ -91,7 +91,7 @@ fn contiguous_box_supports_cpu_sync() { fn streaming_map_has_explicit_device_and_cpu_sync() { let tracker = Box::new(TrackingDmaOp::new().with_next_dma_addr(0x4000)); let tracker = Box::leak(tracker); - let dev = DeviceDma::new(u64::MAX, tracker); + let dev = DeviceDma::new_legacy(u64::MAX, tracker); let mut backing = [0u8; 128]; let map = dev .map_streaming_slice(&mut backing, 64, DmaDirection::Bidirectional) @@ -116,7 +116,7 @@ fn streaming_map_has_explicit_device_and_cpu_sync() { fn streaming_map_for_device_syncs_after_mapping() { let tracker = Box::new(TrackingDmaOp::new().with_next_dma_addr(0x4000)); let tracker = Box::leak(tracker); - let dev = DeviceDma::new(u64::MAX, tracker); + let dev = DeviceDma::new_legacy(u64::MAX, tracker); let mut backing = [0u8; 128]; tracker.clear(); @@ -141,7 +141,7 @@ fn streaming_map_for_device_syncs_after_mapping() { fn streaming_write_for_device_syncs_after_cpu_write() { let tracker = Box::new(TrackingDmaOp::new().with_next_dma_addr(0x4000)); let tracker = Box::leak(tracker); - let dev = DeviceDma::new(u64::MAX, tracker); + let dev = DeviceDma::new_legacy(u64::MAX, tracker); let mut backing = [0u8; 16]; let mut map = dev .map_streaming_slice(&mut backing, 4, DmaDirection::ToDevice) @@ -166,7 +166,7 @@ fn streaming_write_for_device_syncs_after_cpu_write() { fn streaming_read_from_device_syncs_before_cpu_read_and_copies_bounce_buffer() { let tracker = Box::new(TrackingDmaOp::new().with_next_dma_addr(0x80)); let tracker = Box::leak(tracker); - let dev = DeviceDma::new(0xff, tracker); + let dev = DeviceDma::new_legacy(0xff, tracker); let mut backing = [1u8; 16]; let map = dev .map_streaming_slice(&mut backing, 16, DmaDirection::FromDevice) @@ -200,7 +200,7 @@ fn streaming_read_from_device_syncs_before_cpu_read_and_copies_bounce_buffer() { fn streaming_bounce_buffer_copies_back_on_cpu_sync() { let tracker = Box::new(TrackingDmaOp::new().with_next_dma_addr(0x80)); let tracker = Box::leak(tracker); - let dev = DeviceDma::new(0xff, tracker); + let dev = DeviceDma::new_legacy(0xff, tracker); let mut backing = [1u8; 16]; let map = dev .map_streaming_slice(&mut backing, 16, DmaDirection::FromDevice) @@ -298,11 +298,26 @@ fn allocation_rejects_backend_address_outside_mask() { assert!(matches!(result, Err(DmaError::DmaMaskNotMatch { .. }))); } +#[test] +fn explicit_dma_domain_survives_constraint_updates() { + let tracker = Box::new(TrackingDmaOp::new()); + let tracker = Box::leak(tracker); + let domain = DmaDomainId::from_raw(0x42); + let dev = DeviceDma::new(domain, u64::MAX, tracker); + + assert_eq!(dev.domain_id(), domain); + assert_eq!( + dev.with_constraints(DmaConstraints::new(u32::MAX as u64)) + .domain_id(), + domain + ); +} + #[test] fn low_32bit_allocations_are_validated() { let tracker = Box::new(TrackingDmaOp::new().with_next_dma_addr(0xffff_f000)); let tracker = Box::leak(tracker); - let dev = DeviceDma::new(u32::MAX as u64, tracker); + let dev = DeviceDma::new_legacy(u32::MAX as u64, tracker); let buff = dev .contiguous_array_zero_with_align::(0x1000, 0x1000, DmaDirection::ToDevice) .unwrap(); diff --git a/os/arceos/modules/axfs-ng/src/block/runtime/mod.rs b/os/arceos/modules/axfs-ng/src/block/runtime/mod.rs index aa161feaea..79b5330f9b 100644 --- a/os/arceos/modules/axfs-ng/src/block/runtime/mod.rs +++ b/os/arceos/modules/axfs-ng/src/block/runtime/mod.rs @@ -16,9 +16,9 @@ pub use device::{ BlockCompletionMode, BlockDeviceHandle, BlockDrainWake, BlockIrqAction, BlockRuntime, BlockRuntimeConfig, QueueRuntime, RdifBlockDevice, map_blk_err_to_ax_err, }; -pub use dma::DmaBufferGuard; #[cfg(test)] pub use dma::VEC_DMA_OP; +pub use dma::{DmaBufferGuard, RuntimeDmaBuffer, new_owned_dma_buffer}; pub use irq::{BlockIrqBridge, DrainEvents}; pub use pending::{ PendingRequest, PendingTable, PollClaim, PollProgress, RequestKey, RequestState, diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/device.rs b/os/arceos/modules/axfs-ng/src/block_runtime/device.rs index 2a96494838..a62935901f 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/device.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/device.rs @@ -2,15 +2,17 @@ use alloc::{boxed::Box, format, string::String, sync::Arc, vec::Vec}; use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use ax_errno::{AxError, AxResult}; -use dma_api::DmaDirection; +use dma_api::{DeviceDma, DmaDirection, DmaDomainId}; use rdif_block::{ - BlkError, CompletionHint, CompletionSink as RdifCompletionSink, DeviceInfo, IQueue, QueueInfo, - Request, RequestFlags, RequestId, RequestOp, RequestStatus, TransferChunk, TransferPlanner, + BlkError, CompletionHint, CompletionSink as RdifCompletionSink, DeviceInfo, IQueue, + OwnedRequest, QueueHandle, QueueInfo, Request, RequestFlags, RequestId, RequestOp, + RequestPoll as OwnedRequestPoll, RequestStatus, TransferChunk, TransferPlanner, TransferRuntimeCaps, validate_request, }; use super::{ BlockIrqBridge, DmaBufferGuard, DrainEvents, PendingTable, PollClaim, PollProgress, RequestKey, + RuntimeDmaBuffer, new_owned_dma_buffer, }; use crate::os::{ BlockIrqOutcome, BlockIrqRegistration, current_task_id, dma_op, notify_drain, @@ -22,6 +24,11 @@ use crate::os::{ const DEFAULT_MAX_TRANSFER_BYTES: usize = 1024 * 1024; const DEFAULT_SUBMIT_WINDOW: usize = 32; +fn runtime_device_dma(domain: DmaDomainId, dma_mask: u64) -> Result { + let dma_op = dma_op().ok_or(BlkError::Io)?; + Ok(DeviceDma::new(domain, dma_mask, dma_op)) +} + type CompletionRecord = (RequestId, Result<(), BlkError>); static BLOCK_DRAIN_DEVICE_BITS: AtomicU64 = AtomicU64::new(0); @@ -178,15 +185,29 @@ pub struct BlockDeviceHandle { } pub struct QueueRuntime { - queue: Box, + queue: RuntimeQueue, driver_queue_id: usize, info: QueueInfo, planner: TransferPlanner, } +enum RuntimeQueue { + Legacy(Box), + Owned(QueueHandle), +} + +impl RuntimeQueue { + fn info(&self) -> QueueInfo { + match self { + Self::Legacy(queue) => queue.info(), + Self::Owned(queue) => queue.info(), + } + } +} + impl QueueRuntime { - pub fn new( - queue: Box, + fn new( + queue: RuntimeQueue, runtime_queue_id: usize, caps: TransferRuntimeCaps, ) -> Result { @@ -370,6 +391,20 @@ impl BlockDeviceHandle { queues: impl IntoIterator>, bridge: Arc, config: BlockRuntimeConfig, + ) -> Result, BlkError> { + Self::new_runtime( + name, + queues.into_iter().map(RuntimeQueue::Legacy), + bridge, + config, + ) + } + + fn new_runtime( + name: impl Into, + queues: impl IntoIterator, + bridge: Arc, + config: BlockRuntimeConfig, ) -> Result, BlkError> { let caps = TransferRuntimeCaps::new(config.max_transfer_bytes, config.max_segments); let mut driver_queue_map = [None; u64::BITS as usize]; @@ -793,16 +828,7 @@ impl BlockDeviceHandle { loop { let mut queue = self.queues[queue_index].lock(); let info = queue.info; - let mut segments = []; - let request = Request { - op: RequestOp::Flush, - lba: 0, - block_count: 0, - segments: &mut segments, - flags: RequestFlags::NONE, - }; - validate_request(info, &request).map_err(map_blk_err_to_ax_err)?; - match queue.queue.submit_request(request) { + match submit_flush_request(&mut queue, info) { Ok(request_id) => { let key = self .pending @@ -1073,26 +1099,49 @@ impl BlockDeviceHandle { ) -> Result { let chunk_range = chunk.byte_offset..chunk.byte_offset + chunk.byte_len; let src = write_src.map(|src| &src[chunk_range.clone()]); - let dma_op = dma_op().ok_or(BlkError::Io)?; - let mut guard = DmaBufferGuard::new( - dma_op, - info.limits.dma_mask, - chunk.byte_len, - info.limits.dma_alignment.max(1), - direction, - chunk, - src, - )?; - let segments = unsafe { guard.segments_for_submit() }; - let request = Request { - op, - lba: chunk.lba, - block_count: chunk.block_count, - segments, - flags: RequestFlags::NONE, + let dma = runtime_device_dma(info.limits.dma_domain, info.limits.dma_mask)?; + let (request_id, buffer) = match &mut queue.queue { + RuntimeQueue::Legacy(legacy) => { + let mut guard = DmaBufferGuard::new( + &dma, + chunk.byte_len, + info.limits.dma_alignment.max(1), + direction, + chunk, + src, + )?; + let segments = unsafe { guard.segments_for_submit() }; + let request = Request { + op, + lba: chunk.lba, + block_count: chunk.block_count, + segments, + flags: RequestFlags::NONE, + }; + validate_request(info, &request)?; + let request_id = legacy.submit_request(request)?; + (request_id, Some(RuntimeDmaBuffer::Legacy(guard))) + } + RuntimeQueue::Owned(owned) => { + let buffer = new_owned_dma_buffer( + &dma, + chunk.byte_len, + info.limits.dma_alignment.max(1), + direction, + src, + )?; + let request_id = owned + .submit_request(OwnedRequest { + op, + lba: chunk.lba, + block_count: chunk.block_count, + data: Some(buffer.prepare_for_device()), + flags: RequestFlags::NONE, + }) + .map_err(|err| err.error)?; + (request_id, None) + } }; - validate_request(info, &request)?; - let request_id = queue.queue.submit_request(request)?; let key = { let mut pending = self.pending.lock(); if pending.contains_inflight_driver_request(info.id, request_id) { @@ -1101,11 +1150,13 @@ impl BlockDeviceHandle { // request id. Keep the DMA guard alive instead of returning it // to the allocator while a broken driver may still complete // into the submitted buffer. - core::mem::forget(guard); + if let Some(buffer) = buffer { + core::mem::forget(buffer); + } self.poison_driver_contract_violation(); return Err(BlkError::InvalidRequest); } - pending.insert_submitted(info.id, request_id, Some(guard))? + pending.insert_submitted(info.id, request_id, buffer)? }; Ok(WindowEntry { key, @@ -1364,16 +1415,21 @@ fn build_rdif_block_device( drain_wake: Arc, ) -> Result { let name = String::from(block.name()); - let mut queues: Vec> = Vec::new(); - while let Some(queue) = block.interface_mut().create_queue() { - queues.push(queue); + let mut queues = Vec::new(); + while let Some(queue) = block.interface_mut().create_owned_queue() { + queues.push(RuntimeQueue::Owned(queue)); + } + if queues.is_empty() { + while let Some(queue) = block.interface_mut().create_queue() { + queues.push(RuntimeQueue::Legacy(queue)); + } } if queues.is_empty() { return Err(AxError::BadState); } let bridge = Arc::new(BlockIrqBridge::new()); - let device = BlockDeviceHandle::new( + let device = BlockDeviceHandle::new_runtime( name.clone(), queues, bridge, @@ -1387,6 +1443,10 @@ fn build_rdif_block_device( if block.is_irq_enabled() { Ok(registrations) } else { + warn!( + "rdif filesystem block device {name} registered IRQ handler but device did \ + not enable completion IRQ" + ); Err((AxError::Unsupported, registrations)) } }) { @@ -1416,19 +1476,37 @@ fn register_rdif_irq_handlers( ) -> RegisterIrqResult { let irq_sources = block.interface().irq_sources(); if irq_sources.is_empty() { + warn!( + "rdif filesystem block device {} exposes no IRQ source", + block.name() + ); return Err((AxError::Unsupported, Vec::new())); } let mut registrations = Vec::new(); for source in irq_sources { let Some((irq_num, handler)) = block.take_irq_handler(source.id) else { + warn!( + "rdif filesystem block device {} has IRQ source {} but no handler", + block.name(), + source.id + ); return Err((AxError::Unsupported, registrations)); }; let action = BlockIrqAction::new(handler, device.clone(), device_index); match register_shared_block_irq(format!("{}/{}", device.name(), source.id), irq_num, action) { Ok(registration) => registrations.push(registration), - Err(err) => return Err((err, registrations)), + Err(err) => { + warn!( + "rdif filesystem block device {} failed to register IRQ source {} on irq {}: \ + {err:?}", + block.name(), + source.id, + irq_num + ); + return Err((err, registrations)); + } } } Ok(registrations) @@ -1500,7 +1578,17 @@ impl BlockDeviceHandle { request_id: RequestId, ) -> Result { let queue = self.queue_by_runtime_id(queue_id)?; - queue.lock().queue.poll_request(request_id) + let mut queue = queue.lock(); + match &mut queue.queue { + RuntimeQueue::Legacy(legacy) => legacy.poll_request(request_id), + RuntimeQueue::Owned(owned) => match owned.poll_request(request_id)? { + OwnedRequestPoll::Pending => Ok(RequestStatus::Pending), + OwnedRequestPoll::Ready(completed) => { + self.finish_owned_completion(queue_id, completed); + Ok(RequestStatus::Complete) + } + }, + } } fn poll_queue_completions( @@ -1510,14 +1598,78 @@ impl BlockDeviceHandle { ) -> Result, BlkError> { let queue = self.queue_by_runtime_id(queue_id)?; let mut queue = queue.lock(); - let mut sink = CollectCompletionSink::default(); - queue.queue.poll_completions(request_ids, &mut sink)?; - Ok(sink.completions) + match &mut queue.queue { + RuntimeQueue::Legacy(legacy) => { + let mut sink = CollectCompletionSink::default(); + legacy.poll_completions(request_ids, &mut sink)?; + Ok(sink.completions) + } + RuntimeQueue::Owned(owned) => { + let mut completions = Vec::new(); + for &request_id in request_ids { + match owned.poll_request(request_id)? { + OwnedRequestPoll::Pending => {} + OwnedRequestPoll::Ready(completed) => { + let result = completed.result; + let id = completed.id; + self.finish_owned_completion(queue_id, completed); + completions.push((id, result)); + } + } + } + Ok(completions) + } + } } fn queue_by_runtime_id(&self, queue_id: usize) -> Result<&SpinNoIrq, BlkError> { self.queues.get(queue_id).ok_or(BlkError::InvalidRequest) } + + fn finish_owned_completion(&self, queue_id: usize, completed: rdif_block::CompletedRequest) { + let key = self + .pending + .lock() + .matching_driver_keys(queue_id, &[completed.id]) + .into_iter() + .next(); + let Some(key) = key else { + return; + }; + let buffer = completed.data.map(RuntimeDmaBuffer::Owned); + let task_id = self + .pending + .lock() + .complete_with_buffer(key, completed.result, buffer); + self.wake_completed_request(key, task_id); + } +} + +#[cfg(feature = "ext4")] +fn submit_flush_request(queue: &mut QueueRuntime, info: QueueInfo) -> Result { + match &mut queue.queue { + RuntimeQueue::Legacy(legacy) => { + let mut segments = []; + let request = Request { + op: RequestOp::Flush, + lba: 0, + block_count: 0, + segments: &mut segments, + flags: RequestFlags::NONE, + }; + validate_request(info, &request)?; + legacy.submit_request(request) + } + RuntimeQueue::Owned(owned) => owned + .submit_request(OwnedRequest { + op: RequestOp::Flush, + lba: 0, + block_count: 0, + data: None, + flags: RequestFlags::NONE, + }) + .map_err(|err| err.error), + } } #[derive(Default)] diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/dma.rs b/os/arceos/modules/axfs-ng/src/block_runtime/dma.rs index 6cc55cf468..adde8a1bf3 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/dma.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/dma.rs @@ -1,6 +1,9 @@ use alloc::{boxed::Box, vec::Vec}; +use core::num::NonZeroUsize; -use dma_api::{ContiguousArray, DeviceDma, DmaDirection, DmaOp}; +#[cfg(test)] +use dma_api::DmaOp; +use dma_api::{CompletedDma, ContiguousArray, CpuDmaBuffer, DeviceDma, DmaDirection}; use rdif_block::{BlkError, Segment, TransferChunk}; /// Guard retained in the pending table until the request reaches completion. @@ -19,15 +22,13 @@ unsafe impl Send for DmaBufferGuard {} impl DmaBufferGuard { pub fn new( - dma_op: &'static dyn DmaOp, - dma_mask: u64, + dma: &DeviceDma, len: usize, align: usize, direction: DmaDirection, chunk: TransferChunk, src: Option<&[u8]>, ) -> Result { - let dma = DeviceDma::new(dma_mask, dma_op); let mut buffer = dma .contiguous_array_zero_with_align(len.max(1), align.max(1), direction) .map_err(BlkError::from)?; @@ -92,6 +93,49 @@ impl DmaBufferGuard { } } +pub enum RuntimeDmaBuffer { + Legacy(DmaBufferGuard), + Owned(CompletedDma), +} + +impl RuntimeDmaBuffer { + pub fn complete(self, dst: Option<&mut [u8]>) { + match self { + Self::Legacy(guard) => guard.complete(dst), + Self::Owned(buffer) => { + if let Some(dst) = dst { + buffer.copy_from_device_to_slice(dst); + } + } + } + } +} + +pub fn new_owned_dma_buffer( + dma: &DeviceDma, + len: usize, + align: usize, + direction: DmaDirection, + src: Option<&[u8]>, +) -> Result { + let mut buffer = CpuDmaBuffer::new_zero( + dma, + NonZeroUsize::new(len.max(1)).expect("len.max(1) is non-zero"), + align.max(1), + direction, + ) + .map_err(BlkError::from)?; + match direction { + DmaDirection::FromDevice => {} + DmaDirection::ToDevice | DmaDirection::Bidirectional => { + if let Some(src) = src { + buffer.copy_to_device_from_slice(src); + } + } + } + Ok(buffer) +} + #[cfg(test)] pub struct VecDmaOp; diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/mod.rs b/os/arceos/modules/axfs-ng/src/block_runtime/mod.rs index c687567caf..03379c1930 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/mod.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/mod.rs @@ -14,9 +14,11 @@ mod tests { }; use ax_errno::AxError; + use dma_api::{DeviceDma, DmaDomainId}; use rdif_block::{ - BlkError, CompletionHint, DeviceInfo, DriverGeneric, IQueue, Interface, QueueInfo, - QueueLimits, Request, RequestId, RequestOp, RequestStatus, + BlkError, CompletionHint, DeviceInfo, DriverGeneric, IQueue, IQueueOwned, Interface, + OwnedRequest, PollError, QueueHandle, QueueInfo, QueueLimits, Request, RequestId, + RequestOp, RequestPoll, RequestStatus, SubmitError, }; use spin::Mutex as SpinNoIrq; @@ -741,19 +743,13 @@ mod tests { ) .unwrap(); let chunk = planner.plan(0, 512).unwrap().next().unwrap(); - let guard = DmaBufferGuard::new( - &VEC_DMA_OP, - u64::MAX, - 512, - 1, - dma_api::DmaDirection::FromDevice, - chunk, - None, - ) - .unwrap(); + let dma = DeviceDma::new(DmaDomainId::legacy_global(), u64::MAX, &VEC_DMA_OP); + let guard = + DmaBufferGuard::new(&dma, 512, 1, dma_api::DmaDirection::FromDevice, chunk, None) + .unwrap(); let key = pending .lock() - .insert_submitted(0, RequestId::new(4), Some(guard)) + .insert_submitted(0, RequestId::new(4), Some(RuntimeDmaBuffer::Legacy(guard))) .unwrap(); pending.lock().abandon(key); assert_eq!( @@ -1054,6 +1050,7 @@ mod tests { struct MockInterface { name: &'static str, queue: Option>, + owned_queue: Option, info: QueueInfo, } @@ -1063,6 +1060,17 @@ mod tests { Self { name: "mock-rdif", queue: Some(Box::new(queue)), + owned_queue: None, + info, + } + } + + fn new_with_owned(legacy: MockQueue, owned: MockOwnedQueue) -> Self { + let info = legacy.info(); + Self { + name: "mock-rdif", + queue: Some(Box::new(legacy)), + owned_queue: Some(QueueHandle::new(Box::new(owned))), info, } } @@ -1086,6 +1094,120 @@ mod tests { fn create_queue(&mut self) -> Option> { self.queue.take() } + + fn create_owned_queue(&mut self) -> Option { + self.owned_queue.take() + } + } + + struct MockOwnedQueue { + info: QueueInfo, + next: usize, + storage: Vec, + requests: BTreeMap, + submitted: Arc, + } + + struct MockOwnedRequest { + data: Option, + pending_polls_remaining: usize, + } + + impl MockOwnedQueue { + fn new(submitted: Arc) -> Self { + Self { + info: QueueInfo { + id: 0, + device: DeviceInfo::new(16, 512), + limits: QueueLimits { + max_inflight: 8, + max_blocks_per_request: 2, + max_segments: 1, + max_segment_size: 1024, + ..QueueLimits::simple(512, u64::MAX) + }, + }, + next: 1, + storage: (0..16 * 512).map(|idx| (idx / 512) as u8).collect(), + requests: BTreeMap::new(), + submitted, + } + } + } + + impl IQueueOwned for MockOwnedQueue { + fn id(&self) -> usize { + self.info.id + } + + fn info(&self) -> QueueInfo { + self.info + } + + fn submit_request(&mut self, request: OwnedRequest) -> Result { + if let Err(err) = rdif_block::validate_owned_request(self.info, &request) { + return Err(SubmitError::new(err, request)); + } + self.submitted.fetch_add(1, Ordering::AcqRel); + let id = RequestId::new(self.next); + self.next += 1; + let data = if let Some(data) = request.data { + let mut buffer = data.into_cpu_buffer(); + match request.op { + RequestOp::Read => { + let start = request.lba as usize * self.info.device.logical_block_size; + let end = start + + request.block_count as usize * self.info.device.logical_block_size; + unsafe { + buffer.as_mut_slice_cpu()[..end - start] + .copy_from_slice(&self.storage[start..end]); + } + } + RequestOp::Write => { + let start = request.lba as usize * self.info.device.logical_block_size; + let end = start + + request.block_count as usize * self.info.device.logical_block_size; + self.storage[start..end].copy_from_slice(buffer.as_slice_cpu()); + } + _ => {} + } + Some(unsafe { buffer.prepare_for_device().into_in_flight() }) + } else { + None + }; + self.requests.insert( + id, + MockOwnedRequest { + data, + pending_polls_remaining: 1, + }, + ); + Ok(id) + } + + fn poll_request(&mut self, request: RequestId) -> Result { + let req = self + .requests + .get_mut(&request) + .ok_or(PollError::UnknownRequest)?; + if req.pending_polls_remaining > 0 { + req.pending_polls_remaining -= 1; + return Ok(RequestPoll::Pending); + } + let req = self.requests.remove(&request).unwrap(); + let data = req + .data + .map(|data| unsafe { data.complete_after_quiesce() }); + Ok(RequestPoll::Ready(rdif_block::CompletedRequest::new( + request, + Ok(()), + data, + ))) + } + + fn cancel_request(&mut self, request: RequestId) -> Result { + self.poll_request(request) + } } #[test] @@ -1129,6 +1251,30 @@ mod tests { assert_eq!(buf[0], 7); } + #[test] + fn runtime_prefers_owned_queue_from_rdif_interface() { + let _guard = test_task_guard(); + install_test_task_ops(); + let legacy_submits = Arc::new(AtomicUsize::new(0)); + let owned_submits = Arc::new(AtomicUsize::new(0)); + let runtime = BlockRuntime::from_rdif_devices([RdifBlockDevice::new( + "mock-rdif", + None, + Box::new(MockInterface::new_with_owned( + MockQueue::with_submit_counter(0, legacy_submits.clone()), + MockOwnedQueue::new(owned_submits.clone()), + )), + )]); + assert_eq!(runtime.devices().len(), 1); + + let mut buf = alloc::vec![0u8; 512]; + runtime.devices()[0].read_blocks(7, &mut buf).unwrap(); + + assert_eq!(buf[0], 7); + assert_eq!(legacy_submits.load(Ordering::Acquire), 0); + assert_eq!(owned_submits.load(Ordering::Acquire), 1); + } + #[test] fn block_device_queue_hint_drains_pending_request() { let _guard = test_task_guard(); diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/request.rs b/os/arceos/modules/axfs-ng/src/block_runtime/request.rs index ec793c6fdf..95d61c4e16 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/request.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/request.rs @@ -2,7 +2,7 @@ use alloc::{collections::BTreeMap, vec::Vec}; use rdif_block::{BlkError, RequestId}; -use super::DmaBufferGuard; +use super::RuntimeDmaBuffer; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct RuntimeRequestId(usize); @@ -56,14 +56,14 @@ pub struct PendingRequest { submitted: SubmittedRequest, state: RequestState, waiter_task_id: Option, - buffer_guard: Option, + buffer_guard: Option, result: Option>, polling: bool, repoll: bool, } impl PendingRequest { - pub fn submitted(submitted: SubmittedRequest, buffer_guard: Option) -> Self { + pub fn submitted(submitted: SubmittedRequest, buffer_guard: Option) -> Self { Self { submitted, state: RequestState::Submitted, @@ -95,7 +95,7 @@ impl PendingRequest { self.waiter_task_id } - pub fn take_completed_guard(&mut self) -> Option { + pub fn take_completed_guard(&mut self) -> Option { if self.result.is_some() { self.buffer_guard.take() } else { @@ -122,11 +122,22 @@ impl PendingRequest { } fn complete(&mut self, result: Result<(), BlkError>) -> Option { + self.complete_with_buffer(result, None) + } + + fn complete_with_buffer( + &mut self, + result: Result<(), BlkError>, + buffer_guard: Option, + ) -> Option { if self.result.is_some() { return None; } let abandoned = matches!(self.state, RequestState::Abandoned); self.result = Some(result); + if buffer_guard.is_some() { + self.buffer_guard = buffer_guard; + } self.polling = false; self.repoll = false; self.state = if result.is_ok() { @@ -176,7 +187,7 @@ impl PendingTable { &mut self, queue_id: usize, request_id: RequestId, - buffer_guard: Option, + buffer_guard: Option, ) -> Result { if self.contains_inflight_driver_request(queue_id, request_id) { return Err(BlkError::InvalidRequest); @@ -225,6 +236,17 @@ impl PendingTable { .and_then(|request| request.complete(result)) } + pub fn complete_with_buffer( + &mut self, + key: RequestKey, + result: Result<(), BlkError>, + buffer_guard: Option, + ) -> Option { + self.requests + .get_mut(&key) + .and_then(|request| request.complete_with_buffer(result, buffer_guard)) + } + pub fn abandon(&mut self, key: RequestKey) { let remove = self .requests @@ -271,7 +293,7 @@ impl PendingTable { pub fn take_completed( &mut self, key: RequestKey, - ) -> Option<(Result<(), BlkError>, Option)> { + ) -> Option<(Result<(), BlkError>, Option)> { let request = self.requests.get(&key)?; let result = request.result?; let mut request = self.requests.remove(&key)?; diff --git a/platforms/ax-plat-riscv64-sg2002/src/drivers/cvsd.rs b/platforms/ax-plat-riscv64-sg2002/src/drivers/cvsd.rs index df49170543..631d54bd54 100644 --- a/platforms/ax-plat-riscv64-sg2002/src/drivers/cvsd.rs +++ b/platforms/ax-plat-riscv64-sg2002/src/drivers/cvsd.rs @@ -7,7 +7,7 @@ use core::{ use ax_driver::{PlatformDevice, block::PlatformDeviceBlock, probe::OnProbeError}; use rdif_block::{ BlkError, DeviceInfo, DriverGeneric, IQueue, Interface, QueueInfo, QueueLimits, Request, - RequestFlags, RequestId, RequestOp, RequestStatus, validate_request, + RequestId, RequestOp, RequestStatus, validate_request, }; use sg200x_bsp::sdmmc::Sdmmc; @@ -220,16 +220,8 @@ impl Interface for CvsdBlock { fn queue_limits(&self) -> QueueLimits { QueueLimits { - dma_mask: u64::MAX, dma_alignment: 0x1000, - max_inflight: 1, - max_blocks_per_request: 1, - max_segments: 1, - max_segment_size: BLOCK_SIZE, - supported_flags: RequestFlags::NONE, - supports_flush: false, - supports_discard: false, - supports_write_zeroes: false, + ..QueueLimits::simple(BLOCK_SIZE, u64::MAX) } } @@ -268,16 +260,8 @@ unsafe impl IQueue for CvsdQueue { ) }, limits: QueueLimits { - dma_mask: u64::MAX, dma_alignment: 0x1000, - max_inflight: 1, - max_blocks_per_request: 1, - max_segments: 1, - max_segment_size: BLOCK_SIZE, - supported_flags: RequestFlags::NONE, - supports_flush: false, - supports_discard: false, - supports_write_zeroes: false, + ..QueueLimits::simple(BLOCK_SIZE, u64::MAX) }, } }