diff --git a/Cargo.lock b/Cargo.lock index 9d2114353f..4ace446fb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -323,6 +323,7 @@ version = "0.3.1" dependencies = [ "ax-kspin", "ax-std", + "axfs-ng-vfs", ] [[package]] @@ -497,13 +498,13 @@ name = "arm-scmi-rs" version = "0.1.2" dependencies = [ "aarch64-cpu-ext", + "ax-kspin", "bitflags 2.11.1", "log", "mbarrier", "nb", "num-align", "smccc", - "spin 0.10.0", "thiserror 2.0.18", "tock-registers 0.10.1", ] @@ -521,7 +522,7 @@ dependencies = [ "axvisor_api", "log", "numeric-enum-macro", - "spin 0.10.0", + "spin", ] [[package]] @@ -531,13 +532,14 @@ dependencies = [ "aarch64-cpu 11.2.0", "aarch64_sysreg", "ax-errno", + "ax-kspin", "ax-memory-addr", "axaddrspace", "axdevice_base", "axvisor_api", "bitmaps", "log", - "spin 0.10.0", + "spin", "tock-registers 0.10.1", ] @@ -867,11 +869,11 @@ name = "ax-driver-net" version = "0.3.13" dependencies = [ "ax-driver-base", + "ax-kspin", "bitflags 2.11.1", "fxmac_rs", "ixgbe-driver", "log", - "spin 0.9.8", ] [[package]] @@ -945,11 +947,12 @@ dependencies = [ "ax-fs-vfs", "ax-hal", "ax-io", + "ax-kspin", "ax-lazyinit", "axfatfs", "log", "rsext4", - "spin 0.10.0", + "spin", ] [[package]] @@ -958,7 +961,7 @@ version = "0.3.11" dependencies = [ "ax-fs-vfs", "log", - "spin 0.9.8", + "spin", ] [[package]] @@ -984,7 +987,7 @@ dependencies = [ "rsext4", "scope-local", "slab", - "spin 0.10.0", + "spin", "starry-fatfs", ] @@ -994,7 +997,7 @@ version = "0.3.12" dependencies = [ "ax-fs-vfs", "log", - "spin 0.9.8", + "spin", ] [[package]] @@ -1027,7 +1030,7 @@ dependencies = [ "fdt-parser", "heapless 0.9.3", "log", - "spin 0.10.0", + "spin", "toml 1.1.2+spec-1.1.0", ] @@ -1203,7 +1206,7 @@ dependencies = [ "cfg-if", "log", "smoltcp 0.13.1", - "spin 0.10.0", + "spin", ] [[package]] @@ -1218,6 +1221,7 @@ dependencies = [ "ax-fs-ng", "ax-hal", "ax-io", + "ax-kspin", "ax-sync", "ax-task", "axfs-ng-vfs", @@ -1227,11 +1231,10 @@ dependencies = [ "enum_dispatch", "event-listener", "hashbrown 0.16.1", - "lazy_static", "log", "ringbuf", "smoltcp 0.13.1", - "spin 0.10.0", + "spin", ] [[package]] @@ -1266,7 +1269,7 @@ dependencies = [ "ax-kernel-guard", "ax-percpu-macros", "cfg-if", - "spin 0.10.0", + "spin", "x86", ] @@ -1322,7 +1325,7 @@ dependencies = [ "ax-lazyinit", "ax-plat", "log", - "spin 0.10.0", + "spin", ] [[package]] @@ -1462,9 +1465,8 @@ dependencies = [ "ax-task", "bindgen 0.72.1", "flatten_objects", - "lazy_static", "scope-local", - "spin 0.10.0", + "spin", ] [[package]] @@ -1531,7 +1533,7 @@ dependencies = [ "ax-kspin", "ax-lazyinit", "lock_api", - "spin 0.10.0", + "spin", ] [[package]] @@ -1567,7 +1569,7 @@ dependencies = [ "extern-trait", "futures-util", "log", - "spin 0.10.0", + "spin", ] [[package]] @@ -1589,6 +1591,7 @@ version = "0.5.11" dependencies = [ "assert_matches", "ax-errno", + "ax-kspin", "ax-lazyinit", "ax-memory-addr", "ax-memory-set", @@ -1601,7 +1604,6 @@ dependencies = [ "lazy_static", "log", "numeric-enum-macro", - "spin 0.10.0", "x86", ] @@ -1615,7 +1617,7 @@ dependencies = [ "gimli 0.33.0", "log", "paste", - "spin 0.10.0", + "spin", ] [[package]] @@ -1668,6 +1670,7 @@ version = "0.4.10" dependencies = [ "arm_vgic", "ax-errno", + "ax-kspin", "ax-memory-addr", "axaddrspace", "axdevice_base", @@ -1676,7 +1679,6 @@ dependencies = [ "log", "range-alloc-arceos", "riscv_vplic", - "spin 0.10.0", ] [[package]] @@ -1704,6 +1706,7 @@ name = "axfs-ng-vfs" version = "0.4.2" dependencies = [ "ax-errno", + "ax-kspin", "axpoll", "bitflags 2.11.1", "cfg-if", @@ -1711,7 +1714,6 @@ dependencies = [ "inherit-methods-macro", "log", "smallvec", - "spin 0.10.0", ] [[package]] @@ -1790,7 +1792,7 @@ dependencies = [ "sdmmc-protocol", "some-serial", "somehal", - "spin 0.10.0", + "spin", ] [[package]] @@ -1841,7 +1843,7 @@ dependencies = [ "bitflags 2.11.1", "futures", "linux-raw-sys 0.12.1", - "spin 0.10.0", + "spin", "tokio", ] @@ -1949,7 +1951,6 @@ dependencies = [ "extern-trait", "fdt-parser", "hashbrown 0.14.5", - "lazy_static", "log", "prettyplease", "quote", @@ -1957,7 +1958,7 @@ dependencies = [ "rdrive", "riscv_vcpu", "riscv_vplic", - "spin 0.10.0", + "spin", "syn 2.0.117", "tokio", "toml 0.9.12+spec-1.1.0", @@ -1992,6 +1993,7 @@ dependencies = [ "arm_vgic", "ax-cpumask", "ax-errno", + "ax-kspin", "ax-memory-addr", "ax-page-table-entry", "ax-page-table-multiarch", @@ -2006,7 +2008,7 @@ dependencies = [ "log", "loongarch_vcpu", "riscv_vcpu", - "spin 0.10.0", + "spin", "x86_vcpu", ] @@ -2253,7 +2255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a844607dec426282e05649372acdc1170b08ba57879be9d3216184ed6dbfd3b" dependencies = [ "log", - "spin 0.10.0", + "spin", ] [[package]] @@ -2268,7 +2270,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b672b945a3e4f4f40bfd4cd5ee07df9e796a42254ce7cd6d2599ad969244c44a" dependencies = [ - "spin 0.10.0", + "spin", ] [[package]] @@ -2738,6 +2740,7 @@ name = "crab-usb" version = "0.9.3" dependencies = [ "anyhow", + "ax-kspin", "bitflags 2.11.1", "crossbeam", "crossbeam-skiplist", @@ -2750,7 +2753,7 @@ dependencies = [ "mbarrier", "nb", "num_enum", - "spin 0.10.0", + "spin", "thiserror 2.0.18", "tock-registers 0.10.1", "trait-ffi", @@ -3327,11 +3330,11 @@ name = "dma-api" version = "0.7.3" dependencies = [ "aarch64-cpu-ext", + "ax-kspin", "cfg-if", "derive_more", "log", "mbarrier", - "spin 0.10.0", "thiserror 2.0.18", ] @@ -4956,9 +4959,6 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -dependencies = [ - "spin 0.9.8", -] [[package]] name = "leb128fmt" @@ -5123,6 +5123,7 @@ name = "loongarch_vcpu" version = "0.5.3" dependencies = [ "ax-errno", + "ax-kspin", "ax-memory-addr", "ax-page-table-multiarch", "ax-percpu", @@ -5131,7 +5132,6 @@ dependencies = [ "axvisor_api", "log", "loongArch64", - "spin 0.10.0", ] [[package]] @@ -5571,6 +5571,7 @@ checksum = "300e4bdb6b46b592948e700ea1ef24a4296491f6a0ee722b258040abd15a3714" name = "nvme-driver" version = "0.4.2" dependencies = [ + "ax-kspin", "byte-unit", "dma-api", "fdt-edit", @@ -5578,7 +5579,6 @@ dependencies = [ "mmio-api", "pcie", "rd-block", - "spin 0.10.0", "tock-registers 0.10.1", ] @@ -6236,10 +6236,10 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" name = "ramdisk" version = "0.1.1" dependencies = [ + "ax-kspin", "dma-api", "rd-block", "rdif-block", - "spin 0.10.0", ] [[package]] @@ -6543,11 +6543,11 @@ dependencies = [ name = "rdif-serial" version = "0.7.1" dependencies = [ + "ax-kspin", "bitflags 2.11.1", "futures", "heapless 0.9.3", "rdif-base", - "spin 0.10.0", "thiserror 2.0.18", ] @@ -6569,6 +6569,7 @@ dependencies = [ name = "rdrive" version = "0.20.1" dependencies = [ + "ax-kspin", "fdt-edit", "fdt-raw", "log", @@ -6580,7 +6581,7 @@ dependencies = [ "rdif-intc", "rdif-pcie", "rdrive-macros", - "spin 0.10.0", + "spin", "thiserror 2.0.18", ] @@ -6597,11 +6598,11 @@ dependencies = [ name = "realtek-rtl8125" version = "0.2.0" dependencies = [ + "ax-kspin", "dma-api", "log", "mmio-api", "rdif-eth", - "spin 0.10.0", "thiserror 2.0.18", "tock-registers 0.10.1", ] @@ -6862,13 +6863,13 @@ name = "riscv_vplic" version = "0.4.12" dependencies = [ "ax-errno", + "ax-kspin", "axaddrspace", "axdevice_base", "axvisor_api", "bitmaps", "log", "riscv-h", - "spin 0.10.0", ] [[package]] @@ -6934,7 +6935,7 @@ dependencies = [ "mbarrier", "num-align", "rdif-base", - "spin 0.10.0", + "spin", "thiserror 2.0.18", "tock-registers 0.10.1", ] @@ -6953,6 +6954,7 @@ dependencies = [ name = "rockchip-soc" version = "0.3.0" dependencies = [ + "ax-kspin", "bitflags 2.11.1", "dma-api", "enum_dispatch", @@ -6960,7 +6962,7 @@ dependencies = [ "log", "mbarrier", "num-align", - "spin 0.10.0", + "spin", "thiserror 2.0.18", "tock-registers 0.10.1", ] @@ -6970,8 +6972,8 @@ name = "rsext4" version = "0.4.1" dependencies = [ "bitflags 2.11.1", - "lazy_static", "log", + "spin", ] [[package]] @@ -7259,7 +7261,7 @@ version = "0.3.7" dependencies = [ "ax-percpu", "ctor 0.6.3", - "spin 0.10.0", + "spin", ] [[package]] @@ -7449,9 +7451,9 @@ version = "0.1.2" dependencies = [ "ax-dma", "ax-errno", + "ax-kspin", "ax-memory-addr", "log", - "spin 0.10.0", ] [[package]] @@ -7688,6 +7690,7 @@ dependencies = [ name = "some-serial" version = "0.4.1" dependencies = [ + "ax-kspin", "bitflags 2.11.1", "enum_dispatch", "fdt-edit", @@ -7696,7 +7699,6 @@ dependencies = [ "rdif-intc", "rdif-serial", "rdrive", - "spin 0.10.0", "thiserror 2.0.18", "tock-registers 0.10.1", "x86", @@ -7737,7 +7739,7 @@ dependencies = [ "smccc", "some-serial", "somehal-macros", - "spin 0.10.0", + "spin", "syn 2.0.117", "thiserror 2.0.18", "tock-registers 0.10.1", @@ -7761,7 +7763,7 @@ dependencies = [ "rdrive", "someboot", "somehal-macros", - "spin 0.10.0", + "spin", "thiserror 2.0.18", "tock-registers 0.10.1", ] @@ -7776,22 +7778,12 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - [[package]] name = "spin" version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" dependencies = [ "lock_api", + "portable-atomic", ] [[package]] @@ -7889,7 +7881,6 @@ dependencies = [ "inherit-methods-macro", "kernel-elf-parser", "ktracepoint", - "lazy_static", "linux-raw-sys 0.12.1", "lock_api", "num_enum", @@ -7902,7 +7893,7 @@ dependencies = [ "sg2002-tpu", "sg200x-bsp", "slab", - "spin 0.10.0", + "spin", "starry-process", "starry-signal", "starry-vm", @@ -9090,7 +9081,7 @@ dependencies = [ "futures", "log", "num_enum", - "spin 0.10.0", + "spin", "thiserror 2.0.18", ] @@ -9948,6 +9939,7 @@ version = "0.5.9" dependencies = [ "ax-crate-interface", "ax-errno", + "ax-kspin", "ax-memory-addr", "ax-page-table-entry", "axaddrspace", @@ -9962,7 +9954,6 @@ dependencies = [ "numeric-enum-macro", "paste", "raw-cpuid 11.6.0", - "spin 0.10.0", "tock-registers 0.10.1", "x86", "x86_64", diff --git a/Cargo.toml b/Cargo.toml index 39e6a41c4c..d3926c7162 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ members = [ "components/scope-local", "components/someboot", "components/somehal-macros", + "components/spin", "components/starry-process", "components/starry-signal", "components/starry-vm", @@ -371,10 +372,9 @@ schemars = "1.2.1" toml = "1" dma-api = { version = "0.7.3", path = "components/dma-api" } mmio-api = { version = "0.2.2", path = "components/mmio-api" } -lazy_static = { version = "1.5", features = ["spin_no_std"] } lock_api = { version = "0.4", default-features = false } log = "0.4" -spin = "0.10" +spin = { version = "0.10", path = "components/spin" } ostool = { version = "0.18" } uefi = "0.36" fdt-edit = "0.2.3" @@ -383,3 +383,6 @@ thiserror = { version = "2", default-features = false } tock-registers = "0.10" sg200x-bsp = "0.6" x86 = "0.52" + +[patch.crates-io] +spin = { path = "components/spin" } diff --git a/components/arm_vgic/Cargo.toml b/components/arm_vgic/Cargo.toml index 045d2f5027..51ef3cd364 100644 --- a/components/arm_vgic/Cargo.toml +++ b/components/arm_vgic/Cargo.toml @@ -22,6 +22,7 @@ default = [] vgicv3 = [] [dependencies] +ax-kspin.workspace = true axaddrspace = { workspace = true } axdevice_base = { workspace = true } axvisor_api = { workspace = true } diff --git a/components/arm_vgic/src/v3/gits.rs b/components/arm_vgic/src/v3/gits.rs index 50ff92e41d..c584e2ce39 100644 --- a/components/arm_vgic/src/v3/gits.rs +++ b/components/arm_vgic/src/v3/gits.rs @@ -16,12 +16,13 @@ use core::{cell::UnsafeCell, ptr}; +use ax_kspin::SpinNoIrq as Mutex; use ax_memory_addr::PhysAddr; use axaddrspace::{GuestPhysAddr, GuestPhysAddrRange, HostPhysAddr}; use axdevice_base::BaseDeviceOps; use axvisor_api::memory::phys_to_virt; use log::{debug, trace}; -use spin::{Mutex, Once}; +use spin::Once; use super::{ registers::*, diff --git a/components/arm_vgic/src/v3/vgicd.rs b/components/arm_vgic/src/v3/vgicd.rs index b9e1ae930a..a1c4017954 100644 --- a/components/arm_vgic/src/v3/vgicd.rs +++ b/components/arm_vgic/src/v3/vgicd.rs @@ -339,4 +339,4 @@ impl VGicD { } // Todo: move this lock to arceos or axvisor -static GICD_LOCK: spin::Mutex<()> = spin::Mutex::new(()); +static GICD_LOCK: ax_kspin::SpinNoIrq<()> = ax_kspin::SpinNoIrq::new(()); diff --git a/components/arm_vgic/src/v3/vgicr.rs b/components/arm_vgic/src/v3/vgicr.rs index 0d01f950fd..031f9c2a8b 100644 --- a/components/arm_vgic/src/v3/vgicr.rs +++ b/components/arm_vgic/src/v3/vgicr.rs @@ -14,12 +14,13 @@ use core::{cell::UnsafeCell, ptr}; +use ax_kspin::SpinNoIrq as Mutex; use ax_memory_addr::PhysAddr; use axaddrspace::{GuestPhysAddr, GuestPhysAddrRange, HostPhysAddr}; use axdevice_base::BaseDeviceOps; use axvisor_api::memory::phys_to_virt; use log::{debug, trace}; -use spin::{Mutex, Once}; +use spin::Once; use super::{ registers::*, diff --git a/components/arm_vgic/src/vgic.rs b/components/arm_vgic/src/vgic.rs index 62f2ea99ec..0561ec3c01 100644 --- a/components/arm_vgic/src/vgic.rs +++ b/components/arm_vgic/src/vgic.rs @@ -13,8 +13,8 @@ // limitations under the License. use ax_errno::AxResult; +use ax_kspin::SpinNoIrq as Mutex; use axvisor_api::vmm::current_vcpu_id; -use spin::Mutex; use crate::{interrupt::VgicInt, registers::GicRegister, vgicd::Vgicd}; diff --git a/components/axaddrspace/Cargo.toml b/components/axaddrspace/Cargo.toml index 99a8e7b5d8..c1dab5e855 100644 --- a/components/axaddrspace/Cargo.toml +++ b/components/axaddrspace/Cargo.toml @@ -38,6 +38,6 @@ x86 = "0.52" [dev-dependencies] lazy_static = "1.5" -spin = "0.10" +ax-kspin.workspace = true assert_matches = "1.5.0" axin = "0.1.0" diff --git a/components/axaddrspace/tests/test_utils/mod.rs b/components/axaddrspace/tests/test_utils/mod.rs index 48170babf0..578eace9f8 100644 --- a/components/axaddrspace/tests/test_utils/mod.rs +++ b/components/axaddrspace/tests/test_utils/mod.rs @@ -13,12 +13,12 @@ // limitations under the License. use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Mutex; use ax_memory_addr::{PAGE_SIZE_4K as PAGE_SIZE, PhysAddr, VirtAddr}; use ax_page_table_multiarch::PagingHandler; use axaddrspace::{AxMmHal, HostPhysAddr, HostVirtAddr}; use lazy_static::lazy_static; -use spin::Mutex; /// The starting physical address for the simulated memory region in tests. /// This offset is used to map simulated physical addresses to the `MEMORY` array's virtual address space. @@ -124,7 +124,7 @@ pub fn mock_hal_test(test_fn: F) -> R where F: FnOnce() -> R, { - let _guard = TEST_MUTEX.lock(); + let _guard = TEST_MUTEX.lock().unwrap(); MockHal::reset_state(); test_fn() } @@ -170,12 +170,12 @@ impl MockHal { paddr_usize ); let offset = paddr_usize - BASE_PADDR; - VirtAddr::from_usize(MEMORY.lock().0.as_ptr() as usize + offset) + VirtAddr::from_usize(MEMORY.lock().unwrap().0.as_ptr() as usize + offset) } /// Maps a virtual address (within the test process) back to a simulated physical address. pub fn mock_virt_to_phys(vaddr: VirtAddr) -> PhysAddr { - let base_virt = MEMORY.lock().0.as_ptr() as usize; + let base_virt = MEMORY.lock().unwrap().0.as_ptr() as usize; let vaddr_usize = vaddr.as_usize(); assert!( vaddr_usize >= base_virt && vaddr_usize < base_virt + MEMORY_LEN, @@ -199,6 +199,6 @@ impl MockHal { ALLOC_COUNT.store(0, Ordering::SeqCst); DEALLOC_COUNT.store(0, Ordering::SeqCst); // Lock and clear the simulated memory. - MEMORY.lock().0.fill(0); // Fill with zeros to clear any previous test data. + MEMORY.lock().unwrap().0.fill(0); // Fill with zeros to clear any previous test data. } } diff --git a/components/axdevice/Cargo.toml b/components/axdevice/Cargo.toml index 3096b4864a..f1113db5e4 100644 --- a/components/axdevice/Cargo.toml +++ b/components/axdevice/Cargo.toml @@ -17,7 +17,7 @@ license = "Apache-2.0" [dependencies] log = "0.4" cfg-if = "1.0" -spin = "0.10" +ax-kspin.workspace = true # System independent crates provided by ArceOS. ax-errno = { workspace = true } diff --git a/components/axdevice/src/device.rs b/components/axdevice/src/device.rs index b7b08e8692..b53693c16d 100644 --- a/components/axdevice/src/device.rs +++ b/components/axdevice/src/device.rs @@ -18,6 +18,7 @@ use core::ops::Range; #[cfg(target_arch = "aarch64")] use arm_vgic::Vgic; use ax_errno::{AxResult, ax_err}; +use ax_kspin::SpinNoIrq as Mutex; #[cfg(target_arch = "aarch64")] use ax_memory_addr::PhysAddr; use ax_memory_addr::is_aligned_4k; @@ -30,7 +31,6 @@ use axvmconfig::{EmulatedDeviceConfig, EmulatedDeviceType}; use range_alloc_arceos::RangeAllocator; #[cfg(target_arch = "riscv64")] use riscv_vplic::VPlicGlobal; -use spin::Mutex; use crate::AxVmDeviceConfig; diff --git a/components/axdriver_crates/axdriver_net/Cargo.toml b/components/axdriver_crates/axdriver_net/Cargo.toml index 8d41d82ed2..3a0298aca3 100644 --- a/components/axdriver_crates/axdriver_net/Cargo.toml +++ b/components/axdriver_crates/axdriver_net/Cargo.toml @@ -22,8 +22,8 @@ ixgbe = ["dep:ixgbe-driver"] [dependencies] ax-driver-base = { workspace = true } +ax-kspin.workspace = true bitflags = "2.9" fxmac_rs = { workspace = true, optional = true } ixgbe-driver = { version = "0.1.1-preview.1", optional = true } log = { workspace = true } -spin = "0.9" diff --git a/components/axdriver_crates/axdriver_net/src/net_buf.rs b/components/axdriver_crates/axdriver_net/src/net_buf.rs index 3b25606c7f..6251b6e857 100644 --- a/components/axdriver_crates/axdriver_net/src/net_buf.rs +++ b/components/axdriver_crates/axdriver_net/src/net_buf.rs @@ -1,7 +1,7 @@ use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; use core::ptr::NonNull; -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; use crate::{DevError, DevResult}; diff --git a/components/axfs-ng-vfs/Cargo.toml b/components/axfs-ng-vfs/Cargo.toml index eee8cd4431..e4792524fc 100644 --- a/components/axfs-ng-vfs/Cargo.toml +++ b/components/axfs-ng-vfs/Cargo.toml @@ -10,6 +10,7 @@ license = "Apache-2.0" [dependencies] ax-errno = { workspace = true } +ax-kspin = { workspace = true } axpoll = { workspace = true } bitflags = "2.10" cfg-if = "1" @@ -17,4 +18,3 @@ hashbrown = "0.16" inherit-methods-macro = "0.1" log = "0.4" smallvec = "1.15" -spin = { version = "0.10", default-features = false, features = ["mutex"] } diff --git a/components/axfs-ng-vfs/src/lib.rs b/components/axfs-ng-vfs/src/lib.rs index 37b05a3602..6b959e6432 100644 --- a/components/axfs-ng-vfs/src/lib.rs +++ b/components/axfs-ng-vfs/src/lib.rs @@ -16,4 +16,4 @@ pub use types::*; pub type VfsError = ax_errno::AxError; pub type VfsResult = Result; -use spin::{Mutex, MutexGuard}; +use ax_kspin::{SpinNoIrq as Mutex, SpinNoIrqGuard as MutexGuard}; diff --git a/components/axfs-ng-vfs/src/mount.rs b/components/axfs-ng-vfs/src/mount.rs index c3151753c1..6c1be052e9 100644 --- a/components/axfs-ng-vfs/src/mount.rs +++ b/components/axfs-ng-vfs/src/mount.rs @@ -40,7 +40,7 @@ impl Mountpoint { Arc::new(Self { root, location: Mutex::new(location_in_parent), - children: Mutex::default(), + children: Mutex::new(HashMap::default()), device: DEVICE_COUNTER.fetch_add(1, Ordering::Relaxed), }) } diff --git a/components/axfs-ng-vfs/src/node/dir.rs b/components/axfs-ng-vfs/src/node/dir.rs index f57eab1b61..70d449f919 100644 --- a/components/axfs-ng-vfs/src/node/dir.rs +++ b/components/axfs-ng-vfs/src/node/dir.rs @@ -138,8 +138,8 @@ impl DirNode { pub fn new(ops: Arc) -> Self { Self { ops, - cache: Mutex::default(), - mountpoint: Mutex::default(), + cache: Mutex::new(DirChildren::default()), + mountpoint: Mutex::new(None), } } diff --git a/components/axfs-ng-vfs/src/node/mod.rs b/components/axfs-ng-vfs/src/node/mod.rs index 43332db3b1..636010260c 100644 --- a/components/axfs-ng-vfs/src/node/mod.rs +++ b/components/axfs-ng-vfs/src/node/mod.rs @@ -244,7 +244,7 @@ impl DirEntry { node: Node::File(node), node_type, reference, - user_data: Mutex::default(), + user_data: Mutex::new(TypeMap::default()), })) } @@ -253,7 +253,7 @@ impl DirEntry { node: Node::Dir(node_fn(WeakDirEntry(this.clone()))), node_type: NodeType::Directory, reference, - user_data: Mutex::default(), + user_data: Mutex::new(TypeMap::default()), })) } diff --git a/components/axfs_crates/axfs_devfs/Cargo.toml b/components/axfs_crates/axfs_devfs/Cargo.toml index 6f5ae478cd..38d73570b4 100644 --- a/components/axfs_crates/axfs_devfs/Cargo.toml +++ b/components/axfs_crates/axfs_devfs/Cargo.toml @@ -11,5 +11,5 @@ license = "Apache-2.0" [dependencies] ax-fs-vfs.workspace = true -spin = "0.9" +spin = { workspace = true } log = "0.4" diff --git a/components/axfs_crates/axfs_ramfs/Cargo.toml b/components/axfs_crates/axfs_ramfs/Cargo.toml index 3d567e5e80..e63becbb04 100644 --- a/components/axfs_crates/axfs_ramfs/Cargo.toml +++ b/components/axfs_crates/axfs_ramfs/Cargo.toml @@ -11,5 +11,5 @@ license = "Apache-2.0" [dependencies] ax-fs-vfs.workspace = true -spin = "0.9" +spin = { workspace = true } log = "0.4" diff --git a/components/axvm/Cargo.toml b/components/axvm/Cargo.toml index 36d1e8bfe5..46d25cff0f 100644 --- a/components/axvm/Cargo.toml +++ b/components/axvm/Cargo.toml @@ -23,7 +23,7 @@ spin = "0.10" # System independent crates provided by ArceOS. ax-errno = { workspace = true } ax-cpumask = { workspace = true } -# ax-kspin = "0.1.0" +ax-kspin.workspace = true ax-memory-addr = { workspace = true } ax-page-table-entry = { workspace = true, features = ["arm-el2"] } ax-page-table-multiarch = { workspace = true } diff --git a/components/axvm/src/vm.rs b/components/axvm/src/vm.rs index da28e031ae..c3350b3723 100644 --- a/components/axvm/src/vm.rs +++ b/components/axvm/src/vm.rs @@ -17,6 +17,7 @@ use core::{alloc::Layout, fmt}; use ax_cpumask::CpuMask; use ax_errno::{AxError, AxResult, ax_err, ax_err_type}; +use ax_kspin::SpinNoIrq as Mutex; use ax_memory_addr::{align_down_4k, align_up_4k}; use axaddrspace::{ AddrSpace, GuestPhysAddr, HostPhysAddr, HostVirtAddr, MappingFlags, device::AccessWidth, @@ -24,7 +25,7 @@ use axaddrspace::{ use axdevice::{AxVmDeviceConfig, AxVmDevices}; use axvcpu::{AxVCpu, AxVCpuExitReason}; use axvisor_api::vmm::InterruptVector; -use spin::{Mutex, Once}; +use spin::Once; #[cfg(not(target_arch = "x86_64"))] use crate::vcpu::AxVCpuCreateConfig; diff --git a/components/dma-api/Cargo.toml b/components/dma-api/Cargo.toml index 0e8ba67d6e..38c6dd1725 100644 --- a/components/dma-api/Cargo.toml +++ b/components/dma-api/Cargo.toml @@ -12,11 +12,11 @@ version = "0.7.3" [features] [dependencies] +ax-kspin.workspace = true cfg-if.workspace = true derive_more.workspace = true log.workspace = true mbarrier = "0.1" -spin.workspace = true thiserror.workspace = true [target.'cfg(target_arch = "aarch64")'.dependencies] diff --git a/components/dma-api/src/pool.rs b/components/dma-api/src/pool.rs index 9192787fe6..0f6a11395b 100644 --- a/components/dma-api/src/pool.rs +++ b/components/dma-api/src/pool.rs @@ -4,7 +4,7 @@ use alloc::{ }; use core::ops::{Deref, DerefMut}; -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; use crate::{DArray, DeviceDma, DmaDirection, DmaError}; diff --git a/components/loongarch_vcpu/Cargo.toml b/components/loongarch_vcpu/Cargo.toml index 9d8360cbc2..7b26e1c664 100644 --- a/components/loongarch_vcpu/Cargo.toml +++ b/components/loongarch_vcpu/Cargo.toml @@ -13,8 +13,8 @@ license = "Apache-2.0" targets = ["loongarch64-unknown-none-softfloat"] [dependencies] +ax-kspin.workspace = true log = "0.4" -spin = "0.10" ax-errno = { workspace = true } ax-percpu = { workspace = true } diff --git a/components/loongarch_vcpu/src/registers.rs b/components/loongarch_vcpu/src/registers.rs index 47f730e0de..df9baa4448 100644 --- a/components/loongarch_vcpu/src/registers.rs +++ b/components/loongarch_vcpu/src/registers.rs @@ -1,4 +1,4 @@ -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; pub const CSR_GSTAT: u16 = 0x50; pub const CSR_EENTRY: u16 = 0x0c; diff --git a/components/riscv_vplic/Cargo.toml b/components/riscv_vplic/Cargo.toml index f4a1a2f281..11adb4e880 100644 --- a/components/riscv_vplic/Cargo.toml +++ b/components/riscv_vplic/Cargo.toml @@ -15,13 +15,13 @@ license = "Apache-2.0" default = [] [dependencies] +ax-kspin.workspace = true axaddrspace = { workspace = true } axdevice_base = { workspace = true } axvisor_api = { workspace = true } ax-errno = { workspace = true } bitmaps = { version = "3.2", default-features = false } log = "0.4" -spin = "0.10" riscv-h = { workspace = true } [package.metadata.docs.rs] diff --git a/components/riscv_vplic/src/vplic.rs b/components/riscv_vplic/src/vplic.rs index e4ea507dfb..206c3e5a33 100644 --- a/components/riscv_vplic/src/vplic.rs +++ b/components/riscv_vplic/src/vplic.rs @@ -4,9 +4,9 @@ use core::option::Option; +use ax_kspin::SpinNoIrq as Mutex; use axaddrspace::{GuestPhysAddr, HostPhysAddr}; use bitmaps::Bitmap; -use spin::Mutex; use crate::consts::*; diff --git a/components/rsext4/Cargo.toml b/components/rsext4/Cargo.toml index 9124a07735..39772f85ed 100644 --- a/components/rsext4/Cargo.toml +++ b/components/rsext4/Cargo.toml @@ -14,10 +14,9 @@ license = "Apache-2.0" [dependencies] bitflags = "2.10" -lazy_static = { version = "1.5", features = ["spin_no_std"] } log = "0.4" +spin = { workspace = true } [features] default = ["USE_MULTILEVEL_CACHE"] USE_MULTILEVEL_CACHE = [] #启用多级缓存 - diff --git a/components/rsext4/src/crc32c/arm64.rs b/components/rsext4/src/crc32c/arm64.rs index 3f5764370a..5a4413d33e 100644 --- a/components/rsext4/src/crc32c/arm64.rs +++ b/components/rsext4/src/crc32c/arm64.rs @@ -3,10 +3,8 @@ use core::arch::asm; #[cfg(target_arch = "aarch64")] -lazy_static::lazy_static! { - #[allow(dead_code)] - pub static ref HARDWARE_SUPPORT_CRC32: bool = has_hardware_crc32(); -} +#[allow(dead_code)] +pub static HARDWARE_SUPPORT_CRC32: spin::Lazy = spin::Lazy::new(has_hardware_crc32); // In `core::arch::aarch64`, the intrinsics with the `c` suffix implement the // Castagnoli polynomial used by CRC32C. diff --git a/components/spin/CHANGELOG.md b/components/spin/CHANGELOG.md new file mode 100644 index 0000000000..19986ce6b5 --- /dev/null +++ b/components/spin/CHANGELOG.md @@ -0,0 +1,162 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +# Unreleased + +### Added + +### Changed + +### Fixed + +# [0.10.0] - 2025-03-26 + +### Added + +- `Mutex::try_lock_weak` +- `RwLock::try_write_weak` +- `RwLock::try_upgrade_weak` + +### Changed + +- Updated MSRV to 1.60 +- Use `dep:` syntax in Cargo.toml +- `portable_atomic` feature has been renamed to `portable-atomic`, for consistency. + +### Fixed + +# [0.9.8] - 2023-04-03 + +### Fixed + +- Unsoundness in `Once::try_call_once` caused by an `Err(_)` result + +# [0.9.7] - 2023-03-27 + +### Fixed + +- Relaxed accidentally restricted `Send`/`Sync` bounds for `Mutex` guards + +# [0.9.6] - 2023-03-13 + +### Fixed + +- Relaxed accidentally restricted `Send`/`Sync` bounds for `RwLock` guards + +# [0.9.5] - 2023-02-07 + +### Added + +- `FairMutex`, a new mutex implementation that reduces writer starvation. +- A MSRV policy: Rust 1.38 is currently required + +### Changed + +- The crate's CI now has full MIRI integration, further improving the confidence you can have in the implementation. + +### Fixed + +- Ensured that the crate's abstractions comply with stacked borrows rules. +- Unsoundness in the `RwLock` that could be triggered via a reader overflow +- Relaxed various `Send`/`Sync` bound requirements to make the crate more flexible + +# [0.9.4] - 2022-07-14 + +### Fixed + +- Fixed unsoundness in `RwLock` on reader overflow +- Relaxed `Send`/`Sync` bounds for `SpinMutex` and `TicketMutex` (doesn't affect `Mutex` itself) + +# [0.9.3] - 2022-04-17 + +### Added + +- Implemented `Default` for `Once` +- `Once::try_call_once` + +### Fixed + +- Fixed bug that caused `Once::call_once` to incorrectly fail + +# [0.9.2] - 2021-07-09 + +### Changed + +- Improved `Once` performance by reducing the memory footprint of internal state to one byte + +### Fixed + +- Improved performance of `Once` by relaxing ordering guarantees and removing redundant checks + +# [0.9.1] - 2021-06-21 + +### Added + +- Default type parameter on `Once` for better ergonomics + +# [0.9.0] - 2021-03-18 + +### Changed + +- Placed all major API features behind feature flags + +### Fixed + +- A compilation bug with the `lock_api` feature + +# [0.8.0] - 2021-03-15 + +### Added + +- `Once::get_unchecked` +- `RelaxStrategy` trait with type parameter on all locks to support switching between relax strategies + +### Changed + +- `lock_api1` feature is now named `lock_api` + +# [0.7.1] - 2021-01-12 + +### Fixed + +- Prevented `Once` leaking the inner value upon drop + +# [0.7.0] - 2020-10-18 + +### Added + +- `Once::initialized` +- `Once::get_mut` +- `Once::try_into_inner` +- `Once::poll` +- `RwLock`, `Mutex` and `Once` now implement `From` +- `Lazy` type for lazy initialization +- `TicketMutex`, an alternative mutex implementation +- `std` feature flag to enable thread yielding instead of spinning +- `Mutex::is_locked`/`SpinMutex::is_locked`/`TicketMutex::is_locked` +- `Barrier` + +### Changed + +- `Once::wait` now spins even if initialization has not yet started +- `Guard::leak` is now an associated function instead of a method +- Improved the performance of `SpinMutex` by relaxing unnecessarily conservative + ordering requirements + +# [0.6.0] - 2020-10-08 + +### Added + +- More dynamic `Send`/`Sync` bounds for lock guards +- `lock_api` compatibility +- `Guard::leak` methods +- `RwLock::reader_count` and `RwLock::writer_count` +- `Display` implementation for guard types + +### Changed + +- Made `Debug` impls of lock guards just show the inner type like `std` diff --git a/components/spin/Cargo.toml b/components/spin/Cargo.toml new file mode 100644 index 0000000000..6e6a30f344 --- /dev/null +++ b/components/spin/Cargo.toml @@ -0,0 +1,75 @@ +[package] +name = "spin" +version = "0.10.0" +edition = "2021" +authors = [ + "Mathijs van de Nes ", + "John Ericson ", + "Joshua Barretto ", +] +license = "MIT" +repository = "https://github.com/mvdnes/spin-rs.git" +keywords = ["spinlock", "mutex", "rwlock"] +description = "Spin-based synchronization primitives" +rust-version = "1.60" + +[dependencies] +lock_api_crate = { package = "lock_api", version = "0.4", optional = true } +# Enable require-cas feature to provide a better error message if the end user forgets to use the cfg or feature. +portable-atomic = { version = "1.3", optional = true, default-features = false, features = ["require-cas"] } + +[features] +# TGOSKits keeps this crate for `RwLock`, `Once`, and `Lazy` compatibility. +# Non-sleeping mutex users should use `ax-kspin` instead. +# Note: mutex is enabled here for external crates like buddy-slab-allocator +# that still depend on spin::Mutex. This feature should not be used in new code. +default = ["lock_api", "rwlock", "once", "lazy", "mutex"] + +# Enables `Mutex`. By default this uses `SpinMutex`; enable `use_ticket_mutex` +# to make the root `Mutex` alias use `TicketMutex`. +mutex = [] + +# Enables `SpinMutex` and the default spin mutex implementation for `Mutex`. +spin_mutex = ["mutex"] + +# Enables `TicketMutex`. +ticket_mutex = ["mutex"] + +# Enables `FairMutex`. +fair_mutex = ["mutex"] + +# Enables the non-default ticket mutex implementation for `Mutex`. +use_ticket_mutex = ["mutex", "ticket_mutex"] + +# Enables `RwLock`. +rwlock = [] + +# Enables `Once`. +once = [] + +# Enables `Lazy`. +lazy = ["once"] + +# Enables `Barrier`. Because this feature uses `mutex`, either `spin_mutex` or `use_ticket_mutex` must be enabled. +barrier = ["mutex"] + +# Enables `lock_api`-compatible types that use the primitives in this crate internally. +lock_api = ["dep:lock_api_crate"] + +# Enables std-only features such as yield-relaxing. +std = [] + +# Use the `portable-atomic` crate to support platforms without native atomic operations. +# The `portable_atomic_unsafe_assume_single_core` cfg or `critical-section` feature +# of `portable-atomic` crate must also be set by the final binary crate. +# See the documentation for the `portable-atomic` crate for more information +# with some requirements for no-std build: +# https://github.com/taiki-e/portable-atomic#optional-features +portable-atomic = ["dep:portable-atomic"] + +# Deprecated alias: +portable_atomic = ["portable-atomic"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/components/spin/LICENSE b/components/spin/LICENSE new file mode 100644 index 0000000000..b2d7f7bbdc --- /dev/null +++ b/components/spin/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mathijs van de Nes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/components/spin/README.md b/components/spin/README.md new file mode 100644 index 0000000000..80c226fec4 --- /dev/null +++ b/components/spin/README.md @@ -0,0 +1,134 @@ +# spin-rs + +[![Crates.io version](https://img.shields.io/crates/v/spin.svg)](https://crates.io/crates/spin) +[![docs.rs](https://docs.rs/spin/badge.svg)](https://docs.rs/spin/) +[![Build Status](https://travis-ci.org/mvdnes/spin-rs.svg)](https://travis-ci.org/mvdnes/spin-rs) + +Spin-based synchronization primitives. + +This crate provides [spin-based](https://en.wikipedia.org/wiki/Spinlock) +versions of the primitives in `std::sync`. Because synchronization is done +through spinning, the primitives are suitable for use in `no_std` environments. + +Before deciding to use `spin`, we recommend reading +[this superb blog post](https://matklad.github.io/2020/01/02/spinlocks-considered-harmful.html) +by [@matklad](https://github.com/matklad/) that discusses the pros and cons of +spinlocks. If you have access to `std`, it's likely that the primitives in +`std::sync` will serve you better except in very specific circumstances. + +## Features + +- `Mutex`, `RwLock`, `Once`, `Lazy` and `Barrier` equivalents +- Support for `no_std` environments +- [`lock_api`](https://crates.io/crates/lock_api) compatibility +- Upgradeable `RwLock` guards +- Guards can be sent and shared between threads +- Guard leaking +- Ticket locks +- Different strategies for dealing with contention + +## Usage + +Include the following under the `[dependencies]` section in your `Cargo.toml` file. + +```toml +spin = "x.y" +``` + +## Example + +When calling `lock` on a `Mutex` you will get a guard value that provides access +to the data. When this guard is dropped, the mutex will become available again. + +```rust +extern crate spin; +use std::{sync::Arc, thread}; + +fn main() { + let counter = Arc::new(spin::Mutex::new(0)); + + let thread = thread::spawn({ + let counter = counter.clone(); + move || { + for _ in 0..100 { + *counter.lock() += 1; + } + } + }); + + for _ in 0..100 { + *counter.lock() += 1; + } + + thread.join().unwrap(); + + assert_eq!(*counter.lock(), 200); +} +``` + +## Feature flags + +The crate comes with a few feature flags that you may wish to use. + +- `mutex` enables the `Mutex` type. + +- `spin_mutex` enables the `SpinMutex` type. + +- `ticket_mutex` enables the `TicketMutex` type. + +- `use_ticket_mutex` switches to a ticket lock for the implementation of `Mutex`. This + is recommended only on targets for which ordinary spinning locks perform very badly + because it will change the implementation used by other crates that depend on `spin`. + +- `rwlock` enables the `RwLock` type. + +- `once` enables the `Once` type. + +- `lazy` enables the `Lazy` type. + +- `barrier` enables the `Barrier` type. + +- `lock_api` enables support for [`lock_api`](https://crates.io/crates/lock_api) + +- `std` enables support for thread yielding instead of spinning. + +- `portable-atomic` enables usage of the `portable-atomic` crate + to support platforms without native atomic operations (Cortex-M0, etc.). + The `portable_atomic_unsafe_assume_single_core` or `critical-section` feature + of `portable-atomic` crate must also be set by the final binary crate. + See the documentation for the `portable-atomic` crate for more information + with some requirements for no-std build: + https://github.com/taiki-e/portable-atomic#optional-features + +## Remarks + +It is often desirable to have a lock shared between threads. Wrapping the lock in an +`std::sync::Arc` is route through which this might be achieved. + +Locks provide zero-overhead access to their data when accessed through a mutable +reference by using their `get_mut` methods. + +The behaviour of these lock is similar to their namesakes in `std::sync`. they +differ on the following: + +- Locks will not be poisoned in case of failure. +- Threads will not yield to the OS scheduler when encounter a lock that cannot be + accessed. Instead, they will 'spin' in a busy loop until the lock becomes available. + +Many of the feature flags listed above are enabled by default. If you're writing a +library, we recommend disabling those that you don't use to avoid increasing compilation +time for your crate's users. You can do this like so: + +``` +[dependencies] +spin = { version = "x.y", default-features = false, features = [...] } +``` + +## Minimum Safe Rust Version (MSRV) + +This crate is guaranteed to compile on a Minimum Safe Rust Version (MSRV) of 1.60.0 and above. +This version will not be changed without a minor version bump. + +## License + +`spin` is distributed under the MIT License, (See `LICENSE`). diff --git a/components/spin/SECURITY.md b/components/spin/SECURITY.md new file mode 100644 index 0000000000..0dd66d578e --- /dev/null +++ b/components/spin/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are applied only to the latest release. + +## Reporting a Vulnerability + +If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. + +Please disclose it at our [security advisory](https://github.com/mvdnes/spin-rs/security/advisories/new). + +This project is maintained by a team of volunteers on a reasonable-effort basis. As such, vulnerabilities will be disclosed in a best effort base. diff --git a/components/spin/src/barrier.rs b/components/spin/src/barrier.rs new file mode 100644 index 0000000000..49b3c99166 --- /dev/null +++ b/components/spin/src/barrier.rs @@ -0,0 +1,242 @@ +//! Synchronization primitive allowing multiple threads to synchronize the +//! beginning of some computation. +//! +//! Implementation adapted from the 'Barrier' type of the standard library. See: +//! +//! +//! Copyright 2014 The Rust Project Developers. See the COPYRIGHT +//! file at the top-level directory of this distribution and at +//! . +//! +//! Licensed under the Apache License, Version 2.0 > or the MIT license +//! >, at your +//! option. This file may not be copied, modified, or distributed +//! except according to those terms. + +use crate::{RelaxStrategy, Spin, mutex::Mutex}; + +/// A primitive that synchronizes the execution of multiple threads. +/// +/// # Example +/// +/// ``` +/// use std::{sync::Arc, thread}; +/// +/// use spin; +/// +/// let mut handles = Vec::with_capacity(10); +/// let barrier = Arc::new(spin::Barrier::new(10)); +/// for _ in 0..10 { +/// let c = barrier.clone(); +/// // The same messages will be printed together. +/// // You will NOT see any interleaving. +/// handles.push(thread::spawn(move || { +/// println!("before wait"); +/// c.wait(); +/// println!("after wait"); +/// })); +/// } +/// // Wait for other threads to finish. +/// for handle in handles { +/// handle.join().unwrap(); +/// } +/// ``` +pub struct Barrier { + lock: Mutex, + num_threads: usize, +} + +// The inner state of a double barrier +struct BarrierState { + count: usize, + generation_id: usize, +} + +/// A `BarrierWaitResult` is returned by [`wait`] when all threads in the [`Barrier`] +/// have rendezvoused. +/// +/// [`wait`]: struct.Barrier.html#method.wait +/// [`Barrier`]: struct.Barrier.html +/// +/// # Examples +/// +/// ``` +/// use spin; +/// +/// let barrier = spin::Barrier::new(1); +/// let barrier_wait_result = barrier.wait(); +/// ``` +pub struct BarrierWaitResult(bool); + +impl Barrier { + /// Blocks the current thread until all threads have rendezvoused here. + /// + /// Barriers are re-usable after all threads have rendezvoused once, and can + /// be used continuously. + /// + /// A single (arbitrary) thread will receive a [`BarrierWaitResult`] that + /// returns `true` from [`is_leader`] when returning from this function, and + /// all other threads will receive a result that will return `false` from + /// [`is_leader`]. + /// + /// [`BarrierWaitResult`]: struct.BarrierWaitResult.html + /// [`is_leader`]: struct.BarrierWaitResult.html#method.is_leader + /// + /// # Examples + /// + /// ``` + /// use std::{sync::Arc, thread}; + /// + /// use spin; + /// + /// let mut handles = Vec::with_capacity(10); + /// let barrier = Arc::new(spin::Barrier::new(10)); + /// for _ in 0..10 { + /// let c = barrier.clone(); + /// // The same messages will be printed together. + /// // You will NOT see any interleaving. + /// handles.push(thread::spawn(move || { + /// println!("before wait"); + /// c.wait(); + /// println!("after wait"); + /// })); + /// } + /// // Wait for other threads to finish. + /// for handle in handles { + /// handle.join().unwrap(); + /// } + /// ``` + pub fn wait(&self) -> BarrierWaitResult { + let mut lock = self.lock.lock(); + lock.count += 1; + + if lock.count < self.num_threads { + // not the leader + let local_gen = lock.generation_id; + + while local_gen == lock.generation_id && lock.count < self.num_threads { + drop(lock); + R::relax(); + lock = self.lock.lock(); + } + BarrierWaitResult(false) + } else { + // this thread is the leader, + // and is responsible for incrementing the generation + lock.count = 0; + lock.generation_id = lock.generation_id.wrapping_add(1); + BarrierWaitResult(true) + } + } +} + +impl Barrier { + /// Creates a new barrier that can block a given number of threads. + /// + /// A barrier will block `n`-1 threads which call [`wait`] and then wake up + /// all threads at once when the `n`th thread calls [`wait`]. A Barrier created + /// with n = 0 will behave identically to one created with n = 1. + /// + /// [`wait`]: #method.wait + /// + /// # Examples + /// + /// ``` + /// use spin; + /// + /// let barrier = spin::Barrier::new(10); + /// ``` + pub const fn new(n: usize) -> Self { + Self { + lock: Mutex::new(BarrierState { + count: 0, + generation_id: 0, + }), + num_threads: n, + } + } +} + +impl BarrierWaitResult { + /// Returns whether this thread from [`wait`] is the "leader thread". + /// + /// Only one thread will have `true` returned from their result, all other + /// threads will have `false` returned. + /// + /// [`wait`]: struct.Barrier.html#method.wait + /// + /// # Examples + /// + /// ``` + /// use spin; + /// + /// let barrier = spin::Barrier::new(1); + /// let barrier_wait_result = barrier.wait(); + /// println!("{:?}", barrier_wait_result.is_leader()); + /// ``` + pub fn is_leader(&self) -> bool { + self.0 + } +} + +#[cfg(test)] +mod tests { + use std::{ + prelude::v1::*, + sync::{ + Arc, + mpsc::{TryRecvError, channel}, + }, + thread, + }; + + type Barrier = super::Barrier; + + fn use_barrier(n: usize, barrier: Arc) { + let (tx, rx) = channel(); + + let mut ts = Vec::new(); + for _ in 0..n - 1 { + let c = barrier.clone(); + let tx = tx.clone(); + ts.push(thread::spawn(move || { + tx.send(c.wait().is_leader()).unwrap(); + })); + } + + // At this point, all spawned threads should be blocked, + // so we shouldn't get anything from the port + assert!(match rx.try_recv() { + Err(TryRecvError::Empty) => true, + _ => false, + }); + + let mut leader_found = barrier.wait().is_leader(); + + // Now, the barrier is cleared and we should get data. + for _ in 0..n - 1 { + if rx.recv().unwrap() { + assert!(!leader_found); + leader_found = true; + } + } + assert!(leader_found); + + for t in ts { + t.join().unwrap(); + } + } + + #[test] + fn test_barrier() { + const N: usize = 10; + + let barrier = Arc::new(Barrier::new(N)); + + use_barrier(N, barrier.clone()); + + // use barrier twice to ensure it is reusable + use_barrier(N, barrier.clone()); + } +} diff --git a/components/spin/src/lazy.rs b/components/spin/src/lazy.rs new file mode 100644 index 0000000000..ca19777c48 --- /dev/null +++ b/components/spin/src/lazy.rs @@ -0,0 +1,125 @@ +//! Synchronization primitives for lazy evaluation. +//! +//! Implementation adapted from the `SyncLazy` type of the standard library. See: +//! + +use core::{cell::Cell, fmt, ops::Deref}; + +use crate::{RelaxStrategy, Spin, once::Once}; + +/// A value which is initialized on the first access. +/// +/// This type is a thread-safe `Lazy`, and can be used in statics. +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashMap; +/// +/// use spin::Lazy; +/// +/// static HASHMAP: Lazy> = Lazy::new(|| { +/// println!("initializing"); +/// let mut m = HashMap::new(); +/// m.insert(13, "Spica".to_string()); +/// m.insert(74, "Hoyten".to_string()); +/// m +/// }); +/// +/// fn main() { +/// println!("ready"); +/// std::thread::spawn(|| { +/// println!("{:?}", HASHMAP.get(&13)); +/// }) +/// .join() +/// .unwrap(); +/// println!("{:?}", HASHMAP.get(&74)); +/// +/// // Prints: +/// // ready +/// // initializing +/// // Some("Spica") +/// // Some("Hoyten") +/// } +/// ``` +pub struct Lazy T, R = Spin> { + cell: Once, + init: Cell>, +} + +impl fmt::Debug for Lazy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut d = f.debug_tuple("Lazy"); + let d = if let Some(x) = self.cell.get() { + d.field(&x) + } else { + d.field(&format_args!("")) + }; + d.finish() + } +} + +// We never create a `&F` from a `&Lazy` so it is fine +// to not impl `Sync` for `F` +// we do create a `&mut Option` in `force`, but this is +// properly synchronized, so it only happens once +// so it also does not contribute to this impl. +unsafe impl Sync for Lazy where Once: Sync {} +// auto-derived `Send` impl is OK. + +impl Lazy { + /// Creates a new lazy value with the given initializing + /// function. + pub const fn new(f: F) -> Self { + Self { + cell: Once::new(), + init: Cell::new(Some(f)), + } + } + /// Retrieves a mutable pointer to the inner data. + /// + /// This is especially useful when interfacing with low level code or FFI where the caller + /// explicitly knows that it has exclusive access to the inner data. Note that reading from + /// this pointer is UB until initialized or directly written to. + pub fn as_mut_ptr(&self) -> *mut T { + self.cell.as_mut_ptr() + } +} + +impl T, R: RelaxStrategy> Lazy { + /// Forces the evaluation of this lazy value and + /// returns a reference to result. This is equivalent + /// to the `Deref` impl, but is explicit. + /// + /// # Examples + /// + /// ``` + /// use spin::Lazy; + /// + /// let lazy = Lazy::new(|| 92); + /// + /// assert_eq!(Lazy::force(&lazy), &92); + /// assert_eq!(&*lazy, &92); + /// ``` + pub fn force(this: &Self) -> &T { + this.cell.call_once(|| match this.init.take() { + Some(f) => f(), + None => panic!("Lazy instance has previously been poisoned"), + }) + } +} + +impl T, R: RelaxStrategy> Deref for Lazy { + type Target = T; + + fn deref(&self) -> &T { + Self::force(self) + } +} + +impl Default for Lazy T, R> { + /// Creates a new lazy value using `Default` as the initializing function. + fn default() -> Self { + Self::new(T::default) + } +} diff --git a/components/spin/src/lib.rs b/components/spin/src/lib.rs new file mode 100644 index 0000000000..354e5b028a --- /dev/null +++ b/components/spin/src/lib.rs @@ -0,0 +1,234 @@ +#![cfg_attr(all(not(feature = "std"), not(test)), no_std)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![deny(missing_docs)] + +//! This crate provides [spin-based](https://en.wikipedia.org/wiki/Spinlock) versions of the +//! primitives in `std::sync` and `std::lazy`. Because synchronization is done through spinning, +//! the primitives are suitable for use in `no_std` environments. +//! +//! # Features +//! +//! - `Mutex`, `RwLock`, `Once`/`SyncOnceCell`, and `SyncLazy` equivalents +//! +//! - Support for `no_std` environments +//! +//! - [`lock_api`](https://crates.io/crates/lock_api) compatibility +//! +//! - Upgradeable `RwLock` guards +//! +//! - Guards can be sent and shared between threads +//! +//! - Guard leaking +//! +//! - Ticket locks +//! +//! - Different strategies for dealing with contention +//! +//! # Relationship with `std::sync` +//! +//! While `spin` is not a drop-in replacement for `std::sync` (and +//! [should not be considered as such](https://matklad.github.io/2020/01/02/spinlocks-considered-harmful.html)) +//! an effort is made to keep this crate reasonably consistent with `std::sync`. +//! +//! Many of the types defined in this crate have 'additional capabilities' when compared to `std::sync`: +//! +//! - Because spinning does not depend on the thread-driven model of `std::sync`, guards ([`MutexGuard`], +//! [`RwLockReadGuard`], [`RwLockWriteGuard`], etc.) may be sent and shared between threads. +//! +//! - [`RwLockUpgradableGuard`] supports being upgraded into a [`RwLockWriteGuard`]. +//! +//! - Guards support [leaking](https://doc.rust-lang.org/nomicon/leaking.html). +//! +//! - [`Once`] owns the value returned by its `call_once` initializer. +//! +//! - [`RwLock`] supports counting readers and writers. +//! +//! Conversely, the types in this crate do not have some of the features `std::sync` has: +//! +//! - Locks do not track [panic poisoning](https://doc.rust-lang.org/nomicon/poisoning.html). +//! +//! ## Feature flags +//! +//! The crate comes with a few feature flags that you may wish to use. +//! +//! - `lock_api` enables support for [`lock_api`](https://crates.io/crates/lock_api) +//! +//! - `ticket_mutex` uses a ticket lock for the implementation of `Mutex` +//! +//! - `fair_mutex` enables a fairer implementation of `Mutex` that uses eventual fairness to avoid +//! starvation +//! +//! - `std` enables support for thread yielding instead of spinning +//! +//! - `portable-atomic` enables usage of the `portable-atomic` crate +//! to support platforms without native atomic operations (Cortex-M0, etc.). +//! See the documentation for the `portable-atomic` crate for more information +//! with some requirements for no-std build: +//! https://github.com/taiki-e/portable-atomic#optional-features + +#[cfg(any(test, feature = "std"))] +extern crate core; + +#[cfg(feature = "portable-atomic")] +extern crate portable_atomic; + +#[cfg(all( + not(feature = "portable-atomic"), + any(feature = "mutex", feature = "rwlock", feature = "once") +))] +use core::sync::atomic; + +#[cfg(all( + feature = "portable-atomic", + any(feature = "mutex", feature = "rwlock", feature = "once") +))] +use portable_atomic as atomic; + +#[cfg(feature = "barrier")] +#[cfg_attr(docsrs, doc(cfg(feature = "barrier")))] +pub mod barrier; +#[cfg(feature = "lazy")] +#[cfg_attr(docsrs, doc(cfg(feature = "lazy")))] +pub mod lazy; +#[cfg(feature = "mutex")] +#[cfg_attr(docsrs, doc(cfg(feature = "mutex")))] +pub mod mutex; +#[cfg(feature = "once")] +#[cfg_attr(docsrs, doc(cfg(feature = "once")))] +pub mod once; +pub mod relax; +#[cfg(feature = "rwlock")] +#[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] +pub mod rwlock; + +#[cfg(feature = "mutex")] +#[cfg_attr(docsrs, doc(cfg(feature = "mutex")))] +pub use mutex::MutexGuard; +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +pub use relax::Yield; +pub use relax::{RelaxStrategy, Spin}; +#[cfg(feature = "rwlock")] +#[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] +pub use rwlock::RwLockReadGuard; + +// Avoid confusing inference errors by aliasing away the relax strategy parameter. Users that need to use a different +// relax strategy can do so by accessing the types through their fully-qualified path. This is a little bit horrible +// but sadly adding a default type parameter is *still* a breaking change in Rust (for understandable reasons). + +/// A primitive that synchronizes the execution of multiple threads. See [`barrier::Barrier`] for documentation. +/// +/// A note for advanced users: this alias exists to avoid subtle type inference errors due to the default relax +/// strategy type parameter. If you need a non-default relax strategy, use the fully-qualified path. +#[cfg(feature = "barrier")] +#[cfg_attr(docsrs, doc(cfg(feature = "barrier")))] +pub type Barrier = crate::barrier::Barrier; + +/// A value which is initialized on the first access. See [`lazy::Lazy`] for documentation. +/// +/// A note for advanced users: this alias exists to avoid subtle type inference errors due to the default relax +/// strategy type parameter. If you need a non-default relax strategy, use the fully-qualified path. +#[cfg(feature = "lazy")] +#[cfg_attr(docsrs, doc(cfg(feature = "lazy")))] +pub type Lazy T> = crate::lazy::Lazy; + +/// A primitive that synchronizes the execution of multiple threads. See [`mutex::Mutex`] for documentation. +/// +/// A note for advanced users: this alias exists to avoid subtle type inference errors due to the default relax +/// strategy type parameter. If you need a non-default relax strategy, use the fully-qualified path. +#[cfg(feature = "mutex")] +#[cfg_attr(docsrs, doc(cfg(feature = "mutex")))] +pub type Mutex = crate::mutex::Mutex; + +/// A primitive that provides lazy one-time initialization. See [`once::Once`] for documentation. +/// +/// A note for advanced users: this alias exists to avoid subtle type inference errors due to the default relax +/// strategy type parameter. If you need a non-default relax strategy, use the fully-qualified path. +#[cfg(feature = "once")] +#[cfg_attr(docsrs, doc(cfg(feature = "once")))] +pub type Once = crate::once::Once; + +/// A lock that provides data access to either one writer or many readers. See [`rwlock::RwLock`] for documentation. +/// +/// A note for advanced users: this alias exists to avoid subtle type inference errors due to the default relax +/// strategy type parameter. If you need a non-default relax strategy, use the fully-qualified path. +#[cfg(feature = "rwlock")] +#[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] +pub type RwLock = crate::rwlock::RwLock; + +/// A guard that provides immutable data access but can be upgraded to [`RwLockWriteGuard`]. See +/// [`rwlock::RwLockUpgradableGuard`] for documentation. +/// +/// A note for advanced users: this alias exists to avoid subtle type inference errors due to the default relax +/// strategy type parameter. If you need a non-default relax strategy, use the fully-qualified path. +#[cfg(feature = "rwlock")] +#[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] +pub type RwLockUpgradableGuard<'a, T> = crate::rwlock::RwLockUpgradableGuard<'a, T>; + +/// A guard that provides mutable data access. See [`rwlock::RwLockWriteGuard`] for documentation. +/// +/// A note for advanced users: this alias exists to avoid subtle type inference errors due to the default relax +/// strategy type parameter. If you need a non-default relax strategy, use the fully-qualified path. +#[cfg(feature = "rwlock")] +#[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] +pub type RwLockWriteGuard<'a, T> = crate::rwlock::RwLockWriteGuard<'a, T>; + +/// Spin synchronisation primitives, but compatible with [`lock_api`](https://crates.io/crates/lock_api). +#[cfg(feature = "lock_api")] +#[cfg_attr(docsrs, doc(cfg(feature = "lock_api")))] +pub mod lock_api { + /// A lock that provides mutually exclusive data access (compatible with [`lock_api`](https://crates.io/crates/lock_api)). + #[cfg(feature = "mutex")] + #[cfg_attr(docsrs, doc(cfg(feature = "mutex")))] + pub type Mutex = lock_api_crate::Mutex, T>; + + /// A guard that provides mutable data access (compatible with [`lock_api`](https://crates.io/crates/lock_api)). + #[cfg(feature = "mutex")] + #[cfg_attr(docsrs, doc(cfg(feature = "mutex")))] + pub type MutexGuard<'a, T> = lock_api_crate::MutexGuard<'a, crate::Mutex<()>, T>; + + /// A lock that provides data access to either one writer or many readers (compatible with [`lock_api`](https://crates.io/crates/lock_api)). + #[cfg(feature = "rwlock")] + #[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] + pub type RwLock = lock_api_crate::RwLock, T>; + + /// A guard that provides immutable data access (compatible with [`lock_api`](https://crates.io/crates/lock_api)). + #[cfg(feature = "rwlock")] + #[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] + pub type RwLockReadGuard<'a, T> = lock_api_crate::RwLockReadGuard<'a, crate::RwLock<()>, T>; + + /// A guard that provides mutable data access (compatible with [`lock_api`](https://crates.io/crates/lock_api)). + #[cfg(feature = "rwlock")] + #[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] + pub type RwLockWriteGuard<'a, T> = lock_api_crate::RwLockWriteGuard<'a, crate::RwLock<()>, T>; + + /// A guard that provides immutable data access but can be upgraded to [`RwLockWriteGuard`] (compatible with [`lock_api`](https://crates.io/crates/lock_api)). + #[cfg(feature = "rwlock")] + #[cfg_attr(docsrs, doc(cfg(feature = "rwlock")))] + pub type RwLockUpgradableReadGuard<'a, T> = + lock_api_crate::RwLockUpgradableReadGuard<'a, crate::RwLock<()>, T>; +} + +/// In the event of an invalid operation, it's best to abort the current process. +#[cfg(feature = "fair_mutex")] +fn abort() -> ! { + #[cfg(not(feature = "std"))] + { + // Panicking while panicking is defined by Rust to result in an abort. + struct Panic; + + impl Drop for Panic { + fn drop(&mut self) { + panic!("aborting due to invalid operation"); + } + } + + let _panic = Panic; + panic!("aborting due to invalid operation"); + } + + #[cfg(feature = "std")] + { + std::process::abort(); + } +} diff --git a/components/spin/src/mutex.rs b/components/spin/src/mutex.rs new file mode 100644 index 0000000000..918de72690 --- /dev/null +++ b/components/spin/src/mutex.rs @@ -0,0 +1,344 @@ +//! Locks that have the same behaviour as a mutex. +//! +//! The [`Mutex`] in the root of the crate, can be configured using the `ticket_mutex` feature. +//! If it's enabled, [`TicketMutex`] and [`TicketMutexGuard`] will be re-exported as [`Mutex`] +//! and [`MutexGuard`], otherwise the [`SpinMutex`] and guard will be re-exported. +//! +//! `ticket_mutex` is disabled by default. +//! +//! [`Mutex`]: ./struct.Mutex.html +//! [`MutexGuard`]: ./struct.MutexGuard.html +//! [`TicketMutex`]: ./ticket/struct.TicketMutex.html +//! [`TicketMutexGuard`]: ./ticket/struct.TicketMutexGuard.html +//! [`SpinMutex`]: ./spin/struct.SpinMutex.html +//! [`SpinMutexGuard`]: ./spin/struct.SpinMutexGuard.html + +#[cfg(any( + feature = "spin_mutex", + all(feature = "mutex", not(feature = "use_ticket_mutex")) +))] +#[cfg_attr(docsrs, doc(cfg(feature = "spin_mutex")))] +pub mod spin; +#[cfg(any( + feature = "spin_mutex", + all(feature = "mutex", not(feature = "use_ticket_mutex")) +))] +#[cfg_attr(docsrs, doc(cfg(feature = "spin_mutex")))] +pub use self::spin::{SpinMutex, SpinMutexGuard}; + +#[cfg(feature = "ticket_mutex")] +#[cfg_attr(docsrs, doc(cfg(feature = "ticket_mutex")))] +pub mod ticket; +#[cfg(feature = "ticket_mutex")] +#[cfg_attr(docsrs, doc(cfg(feature = "ticket_mutex")))] +pub use self::ticket::{TicketMutex, TicketMutexGuard}; + +#[cfg(feature = "fair_mutex")] +#[cfg_attr(docsrs, doc(cfg(feature = "fair_mutex")))] +pub mod fair; +use core::{ + fmt, + ops::{Deref, DerefMut}, +}; + +#[cfg(feature = "fair_mutex")] +#[cfg_attr(docsrs, doc(cfg(feature = "fair_mutex")))] +pub use self::fair::{FairMutex, FairMutexGuard, Starvation}; +use crate::{RelaxStrategy, Spin}; + +#[cfg(not(feature = "use_ticket_mutex"))] +type InnerMutex = self::spin::SpinMutex; +#[cfg(not(feature = "use_ticket_mutex"))] +type InnerMutexGuard<'a, T> = self::spin::SpinMutexGuard<'a, T>; + +#[cfg(feature = "use_ticket_mutex")] +type InnerMutex = self::ticket::TicketMutex; +#[cfg(feature = "use_ticket_mutex")] +type InnerMutexGuard<'a, T> = self::ticket::TicketMutexGuard<'a, T>; + +/// A spin-based lock providing mutually exclusive access to data. +/// +/// The implementation uses either a ticket mutex or a regular spin mutex depending on whether the `spin_mutex` or +/// `ticket_mutex` feature flag is enabled. +/// +/// # Example +/// +/// ``` +/// use spin; +/// +/// let lock = spin::Mutex::new(0); +/// +/// // Modify the data +/// *lock.lock() = 2; +/// +/// // Read the data +/// let answer = *lock.lock(); +/// assert_eq!(answer, 2); +/// ``` +/// +/// # Thread safety example +/// +/// ``` +/// use std::sync::{Arc, Barrier}; +/// +/// use spin; +/// +/// let thread_count = 1000; +/// let spin_mutex = Arc::new(spin::Mutex::new(0)); +/// +/// // We use a barrier to ensure the readout happens after all writing +/// let barrier = Arc::new(Barrier::new(thread_count + 1)); +/// +/// # let mut ts = Vec::new(); +/// for _ in 0..thread_count { +/// let my_barrier = barrier.clone(); +/// let my_lock = spin_mutex.clone(); +/// # let t = +/// std::thread::spawn(move || { +/// let mut guard = my_lock.lock(); +/// *guard += 1; +/// +/// // Release the lock to prevent a deadlock +/// drop(guard); +/// my_barrier.wait(); +/// }); +/// # ts.push(t); +/// } +/// +/// barrier.wait(); +/// +/// let answer = { *spin_mutex.lock() }; +/// assert_eq!(answer, thread_count); +/// +/// # for t in ts { +/// # t.join().unwrap(); +/// # } +/// ``` +pub struct Mutex { + inner: InnerMutex, +} + +unsafe impl Sync for Mutex {} +unsafe impl Send for Mutex {} + +/// A generic guard that will protect some data access and +/// uses either a ticket lock or a normal spin mutex. +/// +/// For more info see [`TicketMutexGuard`] or [`SpinMutexGuard`]. +/// +/// [`TicketMutexGuard`]: ./struct.TicketMutexGuard.html +/// [`SpinMutexGuard`]: ./struct.SpinMutexGuard.html +pub struct MutexGuard<'a, T: 'a + ?Sized> { + inner: InnerMutexGuard<'a, T>, +} + +impl Mutex { + /// Creates a new [`Mutex`] wrapping the supplied data. + /// + /// # Example + /// + /// ``` + /// use spin::Mutex; + /// + /// static MUTEX: Mutex<()> = Mutex::new(()); + /// + /// fn demo() { + /// let lock = MUTEX.lock(); + /// // do something with lock + /// drop(lock); + /// } + /// ``` + #[inline(always)] + pub const fn new(value: T) -> Self { + Self { + inner: InnerMutex::new(value), + } + } + + /// Consumes this [`Mutex`] and unwraps the underlying data. + /// + /// # Example + /// + /// ``` + /// let lock = spin::Mutex::new(42); + /// assert_eq!(42, lock.into_inner()); + /// ``` + #[inline(always)] + pub fn into_inner(self) -> T { + self.inner.into_inner() + } +} + +impl Mutex { + /// Locks the [`Mutex`] and returns a guard that permits access to the inner data. + /// + /// The returned value may be dereferenced for data access + /// and the lock will be dropped when the guard falls out of scope. + /// + /// ``` + /// let lock = spin::Mutex::new(0); + /// { + /// let mut data = lock.lock(); + /// // The lock is now locked and the data can be accessed + /// *data += 1; + /// // The lock is implicitly dropped at the end of the scope + /// } + /// ``` + #[inline(always)] + pub fn lock(&self) -> MutexGuard<'_, T> { + MutexGuard { + inner: self.inner.lock(), + } + } +} + +impl Mutex { + /// Returns `true` if the lock is currently held. + /// + /// # Safety + /// + /// This function provides no synchronization guarantees and so its result should be considered 'out of date' + /// the instant it is called. Do not use it for synchronization purposes. However, it may be useful as a heuristic. + #[inline(always)] + pub fn is_locked(&self) -> bool { + self.inner.is_locked() + } + + /// Force unlock this [`Mutex`]. + /// + /// # Safety + /// + /// This is *extremely* unsafe if the lock is not held by the current + /// thread. However, this can be useful in some instances for exposing the + /// lock to FFI that doesn't know how to deal with RAII. + #[inline(always)] + pub unsafe fn force_unlock(&self) { + self.inner.force_unlock() + } + + /// Try to lock this [`Mutex`], returning a lock guard if successful. + /// + /// # Example + /// + /// ``` + /// let lock = spin::Mutex::new(42); + /// + /// let maybe_guard = lock.try_lock(); + /// assert!(maybe_guard.is_some()); + /// + /// // `maybe_guard` is still held, so the second call fails + /// let maybe_guard2 = lock.try_lock(); + /// assert!(maybe_guard2.is_none()); + /// ``` + #[inline(always)] + pub fn try_lock(&self) -> Option> { + self.inner + .try_lock() + .map(|guard| MutexGuard { inner: guard }) + } + + /// Returns a mutable reference to the underlying data. + /// + /// Since this call borrows the [`Mutex`] mutably, and a mutable reference is guaranteed to be exclusive in Rust, + /// no actual locking needs to take place -- the mutable borrow statically guarantees no locks exist. As such, + /// this is a 'zero-cost' operation. + /// + /// # Example + /// + /// ``` + /// let mut lock = spin::Mutex::new(0); + /// *lock.get_mut() = 10; + /// assert_eq!(*lock.lock(), 10); + /// ``` + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + self.inner.get_mut() + } +} + +impl fmt::Debug for Mutex { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self.inner, f) + } +} + +impl Default for Mutex { + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl From for Mutex { + fn from(data: T) -> Self { + Self::new(data) + } +} + +impl<'a, T: ?Sized> MutexGuard<'a, T> { + /// Leak the lock guard, yielding a mutable reference to the underlying data. + /// + /// Note that this function will permanently lock the original [`Mutex`]. + /// + /// ``` + /// let mylock = spin::Mutex::new(0); + /// + /// let data: &mut i32 = spin::MutexGuard::leak(mylock.lock()); + /// + /// *data = 1; + /// assert_eq!(*data, 1); + /// ``` + #[inline(always)] + pub fn leak(this: Self) -> &'a mut T { + InnerMutexGuard::leak(this.inner) + } +} + +impl<'a, T: ?Sized + fmt::Debug> fmt::Debug for MutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<'a, T: ?Sized + fmt::Display> fmt::Display for MutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<'a, T: ?Sized> Deref for MutexGuard<'a, T> { + type Target = T; + fn deref(&self) -> &T { + &self.inner + } +} + +impl<'a, T: ?Sized> DerefMut for MutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut T { + &mut self.inner + } +} + +#[cfg(feature = "lock_api")] +unsafe impl lock_api_crate::RawMutex for Mutex<(), R> { + type GuardMarker = lock_api_crate::GuardSend; + + const INIT: Self = Self::new(()); + + fn lock(&self) { + // Prevent guard destructor running + core::mem::forget(Self::lock(self)); + } + + fn try_lock(&self) -> bool { + // Prevent guard destructor running + Self::try_lock(self).map(core::mem::forget).is_some() + } + + unsafe fn unlock(&self) { + self.force_unlock(); + } + + fn is_locked(&self) -> bool { + self.inner.is_locked() + } +} diff --git a/components/spin/src/mutex/fair.rs b/components/spin/src/mutex/fair.rs new file mode 100644 index 0000000000..d744dd2c67 --- /dev/null +++ b/components/spin/src/mutex/fair.rs @@ -0,0 +1,739 @@ +//! A spinning mutex with a fairer unlock algorithm. +//! +//! This mutex is similar to the `SpinMutex` in that it uses spinning to avoid +//! context switches. However, it uses a fairer unlock algorithm that avoids +//! starvation of threads that are waiting for the lock. + +use core::{ + cell::UnsafeCell, + fmt, + marker::PhantomData, + mem::ManuallyDrop, + ops::{Deref, DerefMut}, +}; + +use crate::{ + RelaxStrategy, Spin, + atomic::{AtomicUsize, Ordering}, +}; + +// The lowest bit of `lock` is used to indicate whether the mutex is locked or not. The rest of the bits are used to +// store the number of starving threads. +const LOCKED: usize = 1; +const STARVED: usize = 2; + +/// Number chosen by fair roll of the dice, adjust as needed. +const STARVATION_SPINS: usize = 1024; + +/// A [spin lock](https://en.m.wikipedia.org/wiki/Spinlock) providing mutually exclusive access to data, but with a fairer +/// algorithm. +/// +/// # Example +/// +/// ``` +/// use spin; +/// +/// let lock = spin::mutex::FairMutex::<_>::new(0); +/// +/// // Modify the data +/// *lock.lock() = 2; +/// +/// // Read the data +/// let answer = *lock.lock(); +/// assert_eq!(answer, 2); +/// ``` +/// +/// # Thread safety example +/// +/// ``` +/// use std::sync::{Arc, Barrier}; +/// +/// use spin; +/// +/// let thread_count = 1000; +/// let spin_mutex = Arc::new(spin::mutex::FairMutex::<_>::new(0)); +/// +/// // We use a barrier to ensure the readout happens after all writing +/// let barrier = Arc::new(Barrier::new(thread_count + 1)); +/// +/// for _ in (0..thread_count) { +/// let my_barrier = barrier.clone(); +/// let my_lock = spin_mutex.clone(); +/// std::thread::spawn(move || { +/// let mut guard = my_lock.lock(); +/// *guard += 1; +/// +/// // Release the lock to prevent a deadlock +/// drop(guard); +/// my_barrier.wait(); +/// }); +/// } +/// +/// barrier.wait(); +/// +/// let answer = { *spin_mutex.lock() }; +/// assert_eq!(answer, thread_count); +/// ``` +pub struct FairMutex { + phantom: PhantomData, + pub(crate) lock: AtomicUsize, + data: UnsafeCell, +} + +/// A guard that provides mutable data access. +/// +/// When the guard falls out of scope it will release the lock. +pub struct FairMutexGuard<'a, T: ?Sized + 'a> { + lock: &'a AtomicUsize, + data: *mut T, +} + +/// A handle that indicates that we have been trying to acquire the lock for a while. +/// +/// This handle is used to prevent starvation. +pub struct Starvation<'a, T: ?Sized + 'a, R> { + lock: &'a FairMutex, +} + +/// Indicates whether a lock was rejected due to the lock being held by another thread or due to starvation. +#[derive(Debug)] +pub enum LockRejectReason { + /// The lock was rejected due to the lock being held by another thread. + Locked, + + /// The lock was rejected due to starvation. + Starved, +} + +// Same unsafe impls as `std::sync::Mutex` +unsafe impl Sync for FairMutex {} +unsafe impl Send for FairMutex {} + +unsafe impl Sync for FairMutexGuard<'_, T> {} +unsafe impl Send for FairMutexGuard<'_, T> {} + +impl FairMutex { + /// Creates a new [`FairMutex`] wrapping the supplied data. + /// + /// # Example + /// + /// ``` + /// use spin::mutex::FairMutex; + /// + /// static MUTEX: FairMutex<()> = FairMutex::<_>::new(()); + /// + /// fn demo() { + /// let lock = MUTEX.lock(); + /// // do something with lock + /// drop(lock); + /// } + /// ``` + #[inline(always)] + pub const fn new(data: T) -> Self { + FairMutex { + lock: AtomicUsize::new(0), + data: UnsafeCell::new(data), + phantom: PhantomData, + } + } + + /// Consumes this [`FairMutex`] and unwraps the underlying data. + /// + /// # Example + /// + /// ``` + /// let lock = spin::mutex::FairMutex::<_>::new(42); + /// assert_eq!(42, lock.into_inner()); + /// ``` + #[inline(always)] + pub fn into_inner(self) -> T { + // We know statically that there are no outstanding references to + // `self` so there's no need to lock. + let FairMutex { data, .. } = self; + data.into_inner() + } + + /// Returns a mutable pointer to the underlying data. + /// + /// This is mostly meant to be used for applications which require manual unlocking, but where + /// storing both the lock and the pointer to the inner data gets inefficient. + /// + /// # Example + /// ``` + /// let lock = spin::mutex::FairMutex::<_>::new(42); + /// + /// unsafe { + /// core::mem::forget(lock.lock()); + /// + /// assert_eq!(lock.as_mut_ptr().read(), 42); + /// lock.as_mut_ptr().write(58); + /// + /// lock.force_unlock(); + /// } + /// + /// assert_eq!(*lock.lock(), 58); + /// ``` + #[inline(always)] + pub fn as_mut_ptr(&self) -> *mut T { + self.data.get() + } +} + +impl FairMutex { + /// Locks the [`FairMutex`] and returns a guard that permits access to the inner data. + /// + /// The returned value may be dereferenced for data access + /// and the lock will be dropped when the guard falls out of scope. + /// + /// ``` + /// let lock = spin::mutex::FairMutex::<_>::new(0); + /// { + /// let mut data = lock.lock(); + /// // The lock is now locked and the data can be accessed + /// *data += 1; + /// // The lock is implicitly dropped at the end of the scope + /// } + /// ``` + #[inline(always)] + pub fn lock(&self) -> FairMutexGuard<'_, T> { + // Can fail to lock even if the spinlock is not locked. May be more efficient than `try_lock` + // when called in a loop. + let mut spins = 0; + while self + .lock + .compare_exchange_weak(0, 1, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + // Wait until the lock looks unlocked before retrying + while self.is_locked() { + R::relax(); + + // If we've been spinning for a while, switch to a fairer strategy that will prevent + // newer users from stealing our lock from us. + if spins > STARVATION_SPINS { + return self.starve().lock(); + } + spins += 1; + } + } + + FairMutexGuard { + lock: &self.lock, + data: unsafe { &mut *self.data.get() }, + } + } +} + +impl FairMutex { + /// Returns `true` if the lock is currently held. + /// + /// # Safety + /// + /// This function provides no synchronization guarantees and so its result should be considered 'out of date' + /// the instant it is called. Do not use it for synchronization purposes. However, it may be useful as a heuristic. + #[inline(always)] + pub fn is_locked(&self) -> bool { + self.lock.load(Ordering::Relaxed) & LOCKED != 0 + } + + /// Force unlock this [`FairMutex`]. + /// + /// # Safety + /// + /// This is *extremely* unsafe if the lock is not held by the current + /// thread. However, this can be useful in some instances for exposing the + /// lock to FFI that doesn't know how to deal with RAII. + #[inline(always)] + pub unsafe fn force_unlock(&self) { + self.lock.fetch_and(!LOCKED, Ordering::Release); + } + + /// Try to lock this [`FairMutex`], returning a lock guard if successful. + /// + /// # Example + /// + /// ``` + /// let lock = spin::mutex::FairMutex::<_>::new(42); + /// + /// let maybe_guard = lock.try_lock(); + /// assert!(maybe_guard.is_some()); + /// + /// // `maybe_guard` is still held, so the second call fails + /// let maybe_guard2 = lock.try_lock(); + /// assert!(maybe_guard2.is_none()); + /// ``` + #[inline(always)] + pub fn try_lock(&self) -> Option> { + self.try_lock_starver().ok() + } + + /// Tries to lock this [`FairMutex`] and returns a result that indicates whether the lock was + /// rejected due to a starver or not. + #[inline(always)] + pub fn try_lock_starver(&self) -> Result, LockRejectReason> { + match self + .lock + .compare_exchange(0, LOCKED, Ordering::Acquire, Ordering::Relaxed) + .unwrap_or_else(|x| x) + { + 0 => Ok(FairMutexGuard { + lock: &self.lock, + data: unsafe { &mut *self.data.get() }, + }), + LOCKED => Err(LockRejectReason::Locked), + _ => Err(LockRejectReason::Starved), + } + } + + /// Indicates that the current user has been waiting for the lock for a while + /// and that the lock should yield to this thread over a newly arriving thread. + /// + /// # Example + /// + /// ``` + /// let lock = spin::mutex::FairMutex::<_>::new(42); + /// + /// // Lock the mutex to simulate it being used by another user. + /// let guard1 = lock.lock(); + /// + /// // Try to lock the mutex. + /// let guard2 = lock.try_lock(); + /// assert!(guard2.is_none()); + /// + /// // Wait for a while. + /// wait_for_a_while(); + /// + /// // We are now starved, indicate as such. + /// let starve = lock.starve(); + /// + /// // Once the lock is released, another user trying to lock it will + /// // fail. + /// drop(guard1); + /// let guard3 = lock.try_lock(); + /// assert!(guard3.is_none()); + /// + /// // However, we will be able to lock it. + /// let guard4 = starve.try_lock(); + /// assert!(guard4.is_ok()); + /// + /// # fn wait_for_a_while() {} + /// ``` + pub fn starve(&self) -> Starvation<'_, T, R> { + // Add a new starver to the state. + if self.lock.fetch_add(STARVED, Ordering::Relaxed) > (isize::MAX - 1) as usize { + // In the event of a potential lock overflow, abort. + crate::abort(); + } + + Starvation { lock: self } + } + + /// Returns a mutable reference to the underlying data. + /// + /// Since this call borrows the [`FairMutex`] mutably, and a mutable reference is guaranteed to be exclusive in + /// Rust, no actual locking needs to take place -- the mutable borrow statically guarantees no locks exist. As + /// such, this is a 'zero-cost' operation. + /// + /// # Example + /// + /// ``` + /// let mut lock = spin::mutex::FairMutex::<_>::new(0); + /// *lock.get_mut() = 10; + /// assert_eq!(*lock.lock(), 10); + /// ``` + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + // We know statically that there are no other references to `self`, so + // there's no need to lock the inner mutex. + unsafe { &mut *self.data.get() } + } +} + +impl fmt::Debug for FairMutex { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + struct LockWrapper<'a, T: ?Sized + fmt::Debug>(Option>); + + impl fmt::Debug for LockWrapper<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.0 { + Some(guard) => fmt::Debug::fmt(guard, f), + None => f.write_str(""), + } + } + } + + f.debug_struct("FairMutex") + .field("data", &LockWrapper(self.try_lock())) + .finish() + } +} + +impl Default for FairMutex { + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl From for FairMutex { + fn from(data: T) -> Self { + Self::new(data) + } +} + +impl<'a, T: ?Sized> FairMutexGuard<'a, T> { + /// Leak the lock guard, yielding a mutable reference to the underlying data. + /// + /// Note that this function will permanently lock the original [`FairMutex`]. + /// + /// ``` + /// let mylock = spin::mutex::FairMutex::<_>::new(0); + /// + /// let data: &mut i32 = spin::mutex::FairMutexGuard::leak(mylock.lock()); + /// + /// *data = 1; + /// assert_eq!(*data, 1); + /// ``` + #[inline(always)] + pub fn leak(this: Self) -> &'a mut T { + // Use ManuallyDrop to avoid stacked-borrow invalidation + let mut this = ManuallyDrop::new(this); + // We know statically that only we are referencing data + unsafe { &mut *this.data } + } +} + +impl<'a, T: ?Sized + fmt::Debug> fmt::Debug for FairMutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<'a, T: ?Sized + fmt::Display> fmt::Display for FairMutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<'a, T: ?Sized> Deref for FairMutexGuard<'a, T> { + type Target = T; + fn deref(&self) -> &T { + // We know statically that only we are referencing data + unsafe { &*self.data } + } +} + +impl<'a, T: ?Sized> DerefMut for FairMutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut T { + // We know statically that only we are referencing data + unsafe { &mut *self.data } + } +} + +impl<'a, T: ?Sized> Drop for FairMutexGuard<'a, T> { + /// The dropping of the MutexGuard will release the lock it was created from. + fn drop(&mut self) { + self.lock.fetch_and(!LOCKED, Ordering::Release); + } +} + +impl<'a, T: ?Sized, R> Starvation<'a, T, R> { + /// Attempts the lock the mutex if we are the only starving user. + /// + /// This allows another user to lock the mutex if they are starving as well. + pub fn try_lock_fair(self) -> Result, Self> { + // Try to lock the mutex. + if self + .lock + .lock + .compare_exchange( + STARVED, + STARVED | LOCKED, + Ordering::Acquire, + Ordering::Relaxed, + ) + .is_ok() + { + // We are the only starving user, lock the mutex. + Ok(FairMutexGuard { + lock: &self.lock.lock, + data: self.lock.data.get(), + }) + } else { + // Another user is starving, fail. + Err(self) + } + } + + /// Attempts to lock the mutex. + /// + /// If the lock is currently held by another thread, this will return `None`. + /// + /// # Example + /// + /// ``` + /// let lock = spin::mutex::FairMutex::<_>::new(42); + /// + /// // Lock the mutex to simulate it being used by another user. + /// let guard1 = lock.lock(); + /// + /// // Try to lock the mutex. + /// let guard2 = lock.try_lock(); + /// assert!(guard2.is_none()); + /// + /// // Wait for a while. + /// wait_for_a_while(); + /// + /// // We are now starved, indicate as such. + /// let starve = lock.starve(); + /// + /// // Once the lock is released, another user trying to lock it will + /// // fail. + /// drop(guard1); + /// let guard3 = lock.try_lock(); + /// assert!(guard3.is_none()); + /// + /// // However, we will be able to lock it. + /// let guard4 = starve.try_lock(); + /// assert!(guard4.is_ok()); + /// + /// # fn wait_for_a_while() {} + /// ``` + pub fn try_lock(self) -> Result, Self> { + // Try to lock the mutex. + if self.lock.lock.fetch_or(LOCKED, Ordering::Acquire) & LOCKED == 0 { + // We have successfully locked the mutex. + // By dropping `self` here, we decrement the starvation count. + Ok(FairMutexGuard { + lock: &self.lock.lock, + data: self.lock.data.get(), + }) + } else { + Err(self) + } + } +} + +impl<'a, T: ?Sized, R: RelaxStrategy> Starvation<'a, T, R> { + /// Locks the mutex. + pub fn lock(mut self) -> FairMutexGuard<'a, T> { + // Try to lock the mutex. + loop { + match self.try_lock() { + Ok(lock) => return lock, + Err(starve) => self = starve, + } + + // Relax until the lock is released. + while self.lock.is_locked() { + R::relax(); + } + } + } +} + +impl<'a, T: ?Sized, R> Drop for Starvation<'a, T, R> { + fn drop(&mut self) { + // As there is no longer a user being starved, we decrement the starver count. + self.lock.lock.fetch_sub(STARVED, Ordering::Release); + } +} + +impl fmt::Display for LockRejectReason { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LockRejectReason::Locked => write!(f, "locked"), + LockRejectReason::Starved => write!(f, "starved"), + } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for LockRejectReason {} + +#[cfg(feature = "lock_api")] +unsafe impl lock_api_crate::RawMutex for FairMutex<(), R> { + type GuardMarker = lock_api_crate::GuardSend; + + const INIT: Self = Self::new(()); + + fn lock(&self) { + // Prevent guard destructor running + core::mem::forget(Self::lock(self)); + } + + fn try_lock(&self) -> bool { + // Prevent guard destructor running + Self::try_lock(self).map(core::mem::forget).is_some() + } + + unsafe fn unlock(&self) { + self.force_unlock(); + } + + fn is_locked(&self) -> bool { + Self::is_locked(self) + } +} + +#[cfg(test)] +mod tests { + use std::{ + prelude::v1::*, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + mpsc::channel, + }, + thread, + }; + + type FairMutex = super::FairMutex; + + #[derive(Eq, PartialEq, Debug)] + struct NonCopy(i32); + + #[test] + fn smoke() { + let m = FairMutex::<_>::new(()); + drop(m.lock()); + drop(m.lock()); + } + + #[test] + fn lots_and_lots() { + static M: FairMutex<()> = FairMutex::<_>::new(()); + static mut CNT: u32 = 0; + const J: u32 = 1000; + const K: u32 = 3; + + fn inc() { + for _ in 0..J { + unsafe { + let _g = M.lock(); + CNT += 1; + } + } + } + + let (tx, rx) = channel(); + for _ in 0..K { + let tx2 = tx.clone(); + thread::spawn(move || { + inc(); + tx2.send(()).unwrap(); + }); + let tx2 = tx.clone(); + thread::spawn(move || { + inc(); + tx2.send(()).unwrap(); + }); + } + + drop(tx); + for _ in 0..2 * K { + rx.recv().unwrap(); + } + assert_eq!(unsafe { CNT }, J * K * 2); + } + + #[test] + fn try_lock() { + let mutex = FairMutex::<_>::new(42); + + // First lock succeeds + let a = mutex.try_lock(); + assert_eq!(a.as_ref().map(|r| **r), Some(42)); + + // Additional lock fails + let b = mutex.try_lock(); + assert!(b.is_none()); + + // After dropping lock, it succeeds again + ::core::mem::drop(a); + let c = mutex.try_lock(); + assert_eq!(c.as_ref().map(|r| **r), Some(42)); + } + + #[test] + fn test_into_inner() { + let m = FairMutex::<_>::new(NonCopy(10)); + assert_eq!(m.into_inner(), NonCopy(10)); + } + + #[test] + fn test_into_inner_drop() { + struct Foo(Arc); + impl Drop for Foo { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let num_drops = Arc::new(AtomicUsize::new(0)); + let m = FairMutex::<_>::new(Foo(num_drops.clone())); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + { + let _inner = m.into_inner(); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + } + assert_eq!(num_drops.load(Ordering::SeqCst), 1); + } + + #[test] + fn test_mutex_arc_nested() { + // Tests nested mutexes and access + // to underlying data. + let arc = Arc::new(FairMutex::<_>::new(1)); + let arc2 = Arc::new(FairMutex::<_>::new(arc)); + let (tx, rx) = channel(); + let _t = thread::spawn(move || { + let lock = arc2.lock(); + let lock2 = lock.lock(); + assert_eq!(*lock2, 1); + tx.send(()).unwrap(); + }); + rx.recv().unwrap(); + } + + #[test] + fn test_mutex_arc_access_in_unwind() { + let arc = Arc::new(FairMutex::<_>::new(1)); + let arc2 = arc.clone(); + let _ = thread::spawn(move || -> () { + struct Unwinder { + i: Arc>, + } + impl Drop for Unwinder { + fn drop(&mut self) { + *self.i.lock() += 1; + } + } + let _u = Unwinder { i: arc2 }; + panic!(); + }) + .join(); + let lock = arc.lock(); + assert_eq!(*lock, 2); + } + + #[test] + fn test_mutex_unsized() { + let mutex: &FairMutex<[i32]> = &FairMutex::<_>::new([1, 2, 3]); + { + let b = &mut *mutex.lock(); + b[0] = 4; + b[2] = 5; + } + let comp: &[i32] = &[4, 2, 5]; + assert_eq!(&*mutex.lock(), comp); + } + + #[test] + fn test_mutex_force_lock() { + let lock = FairMutex::<_>::new(()); + ::std::mem::forget(lock.lock()); + unsafe { + lock.force_unlock(); + } + assert!(lock.try_lock().is_some()); + } +} diff --git a/components/spin/src/mutex/spin.rs b/components/spin/src/mutex/spin.rs new file mode 100644 index 0000000000..0f67fbde04 --- /dev/null +++ b/components/spin/src/mutex/spin.rs @@ -0,0 +1,561 @@ +//! A naïve spinning mutex. +//! +//! Waiting threads hammer an atomic variable until it becomes available. Best-case latency is low, but worst-case +//! latency is theoretically infinite. + +use core::{ + cell::UnsafeCell, + fmt, + marker::PhantomData, + mem::ManuallyDrop, + ops::{Deref, DerefMut}, +}; + +use crate::{ + RelaxStrategy, Spin, + atomic::{AtomicBool, Ordering}, +}; + +/// A [spin lock](https://en.m.wikipedia.org/wiki/Spinlock) providing mutually exclusive access to data. +/// +/// # Example +/// +/// ``` +/// use spin; +/// +/// let lock = spin::mutex::SpinMutex::<_>::new(0); +/// +/// // Modify the data +/// *lock.lock() = 2; +/// +/// // Read the data +/// let answer = *lock.lock(); +/// assert_eq!(answer, 2); +/// ``` +/// +/// # Thread safety example +/// +/// ``` +/// use std::sync::{Arc, Barrier}; +/// +/// use spin; +/// +/// let thread_count = 1000; +/// let spin_mutex = Arc::new(spin::mutex::SpinMutex::<_>::new(0)); +/// +/// // We use a barrier to ensure the readout happens after all writing +/// let barrier = Arc::new(Barrier::new(thread_count + 1)); +/// +/// # let mut ts = Vec::new(); +/// for _ in (0..thread_count) { +/// let my_barrier = barrier.clone(); +/// let my_lock = spin_mutex.clone(); +/// # let t = +/// std::thread::spawn(move || { +/// let mut guard = my_lock.lock(); +/// *guard += 1; +/// +/// // Release the lock to prevent a deadlock +/// drop(guard); +/// my_barrier.wait(); +/// }); +/// # ts.push(t); +/// } +/// +/// barrier.wait(); +/// +/// let answer = { *spin_mutex.lock() }; +/// assert_eq!(answer, thread_count); +/// +/// # for t in ts { +/// # t.join().unwrap(); +/// # } +/// ``` +pub struct SpinMutex { + phantom: PhantomData, + pub(crate) lock: AtomicBool, + data: UnsafeCell, +} + +/// A guard that provides mutable data access. +/// +/// When the guard falls out of scope it will release the lock. +pub struct SpinMutexGuard<'a, T: ?Sized + 'a> { + lock: &'a AtomicBool, + data: *mut T, +} + +// Same unsafe impls as `std::sync::Mutex` +unsafe impl Sync for SpinMutex {} +unsafe impl Send for SpinMutex {} + +unsafe impl Sync for SpinMutexGuard<'_, T> {} +unsafe impl Send for SpinMutexGuard<'_, T> {} + +impl SpinMutex { + /// Creates a new [`SpinMutex`] wrapping the supplied data. + /// + /// # Example + /// + /// ``` + /// use spin::mutex::SpinMutex; + /// + /// static MUTEX: SpinMutex<()> = SpinMutex::<_>::new(()); + /// + /// fn demo() { + /// let lock = MUTEX.lock(); + /// // do something with lock + /// drop(lock); + /// } + /// ``` + #[inline(always)] + pub const fn new(data: T) -> Self { + SpinMutex { + lock: AtomicBool::new(false), + data: UnsafeCell::new(data), + phantom: PhantomData, + } + } + + /// Consumes this [`SpinMutex`] and unwraps the underlying data. + /// + /// # Example + /// + /// ``` + /// let lock = spin::mutex::SpinMutex::<_>::new(42); + /// assert_eq!(42, lock.into_inner()); + /// ``` + #[inline(always)] + pub fn into_inner(self) -> T { + // We know statically that there are no outstanding references to + // `self` so there's no need to lock. + let SpinMutex { data, .. } = self; + data.into_inner() + } + + /// Returns a mutable pointer to the underlying data. + /// + /// This is mostly meant to be used for applications which require manual unlocking, but where + /// storing both the lock and the pointer to the inner data gets inefficient. + /// + /// # Example + /// ``` + /// let lock = spin::mutex::SpinMutex::<_>::new(42); + /// + /// unsafe { + /// core::mem::forget(lock.lock()); + /// + /// assert_eq!(lock.as_mut_ptr().read(), 42); + /// lock.as_mut_ptr().write(58); + /// + /// lock.force_unlock(); + /// } + /// + /// assert_eq!(*lock.lock(), 58); + /// ``` + #[inline(always)] + pub fn as_mut_ptr(&self) -> *mut T { + self.data.get() + } +} + +impl SpinMutex { + /// Locks the [`SpinMutex`] and returns a guard that permits access to the inner data. + /// + /// The returned value may be dereferenced for data access + /// and the lock will be dropped when the guard falls out of scope. + /// + /// ``` + /// let lock = spin::mutex::SpinMutex::<_>::new(0); + /// { + /// let mut data = lock.lock(); + /// // The lock is now locked and the data can be accessed + /// *data += 1; + /// // The lock is implicitly dropped at the end of the scope + /// } + /// ``` + #[inline(always)] + pub fn lock(&self) -> SpinMutexGuard<'_, T> { + // Can fail to lock even if the spinlock is not locked. May be more efficient than `try_lock` + // when called in a loop. + loop { + if let Some(guard) = self.try_lock_weak() { + break guard; + } + + while self.is_locked() { + R::relax(); + } + } + } +} + +impl SpinMutex { + /// Returns `true` if the lock is currently held. + /// + /// # Safety + /// + /// This function provides no synchronization guarantees and so its result should be considered 'out of date' + /// the instant it is called. Do not use it for synchronization purposes. However, it may be useful as a heuristic. + #[inline(always)] + pub fn is_locked(&self) -> bool { + self.lock.load(Ordering::Relaxed) + } + + /// Force unlock this [`SpinMutex`]. + /// + /// # Safety + /// + /// This is *extremely* unsafe if the lock is not held by the current + /// thread. However, this can be useful in some instances for exposing the + /// lock to FFI that doesn't know how to deal with RAII. + #[inline(always)] + pub unsafe fn force_unlock(&self) { + self.lock.store(false, Ordering::Release); + } + + /// Try to lock this [`SpinMutex`], returning a lock guard if successful. + /// + /// # Example + /// + /// ``` + /// let lock = spin::mutex::SpinMutex::<_>::new(42); + /// + /// let maybe_guard = lock.try_lock(); + /// assert!(maybe_guard.is_some()); + /// + /// // `maybe_guard` is still held, so the second call fails + /// let maybe_guard2 = lock.try_lock(); + /// assert!(maybe_guard2.is_none()); + /// ``` + #[inline(always)] + pub fn try_lock(&self) -> Option> { + // The reason for using a strong compare_exchange is explained here: + // https://github.com/Amanieu/parking_lot/pull/207#issuecomment-575869107 + if self + .lock + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + Some(SpinMutexGuard { + lock: &self.lock, + data: unsafe { &mut *self.data.get() }, + }) + } else { + None + } + } + + /// Try to lock this [`SpinMutex`], returning a lock guard if succesful. + /// + /// Unlike [`SpinMutex::try_lock`], this function is allowed to spuriously fail even when the mutex is unlocked, + /// which can result in more efficient code on some platforms. + #[inline(always)] + pub fn try_lock_weak(&self) -> Option> { + if self + .lock + .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + Some(SpinMutexGuard { + lock: &self.lock, + data: unsafe { &mut *self.data.get() }, + }) + } else { + None + } + } + + /// Returns a mutable reference to the underlying data. + /// + /// Since this call borrows the [`SpinMutex`] mutably, and a mutable reference is guaranteed to be exclusive in + /// Rust, no actual locking needs to take place -- the mutable borrow statically guarantees no locks exist. As + /// such, this is a 'zero-cost' operation. + /// + /// # Example + /// + /// ``` + /// let mut lock = spin::mutex::SpinMutex::<_>::new(0); + /// *lock.get_mut() = 10; + /// assert_eq!(*lock.lock(), 10); + /// ``` + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + // We know statically that there are no other references to `self`, so + // there's no need to lock the inner mutex. + unsafe { &mut *self.data.get() } + } +} + +impl fmt::Debug for SpinMutex { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.try_lock() { + Some(guard) => write!(f, "Mutex {{ data: ") + .and_then(|()| (*guard).fmt(f)) + .and_then(|()| write!(f, " }}")), + None => write!(f, "Mutex {{ }}"), + } + } +} + +impl Default for SpinMutex { + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl From for SpinMutex { + fn from(data: T) -> Self { + Self::new(data) + } +} + +impl<'a, T: ?Sized> SpinMutexGuard<'a, T> { + /// Leak the lock guard, yielding a mutable reference to the underlying data. + /// + /// Note that this function will permanently lock the original [`SpinMutex`]. + /// + /// ``` + /// let mylock = spin::mutex::SpinMutex::<_>::new(0); + /// + /// let data: &mut i32 = spin::mutex::SpinMutexGuard::leak(mylock.lock()); + /// + /// *data = 1; + /// assert_eq!(*data, 1); + /// ``` + #[inline(always)] + pub fn leak(this: Self) -> &'a mut T { + // Use ManuallyDrop to avoid stacked-borrow invalidation + let mut this = ManuallyDrop::new(this); + // We know statically that only we are referencing data + unsafe { &mut *this.data } + } +} + +impl<'a, T: ?Sized + fmt::Debug> fmt::Debug for SpinMutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<'a, T: ?Sized + fmt::Display> fmt::Display for SpinMutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<'a, T: ?Sized> Deref for SpinMutexGuard<'a, T> { + type Target = T; + fn deref(&self) -> &T { + // We know statically that only we are referencing data + unsafe { &*self.data } + } +} + +impl<'a, T: ?Sized> DerefMut for SpinMutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut T { + // We know statically that only we are referencing data + unsafe { &mut *self.data } + } +} + +impl<'a, T: ?Sized> Drop for SpinMutexGuard<'a, T> { + /// The dropping of the MutexGuard will release the lock it was created from. + fn drop(&mut self) { + self.lock.store(false, Ordering::Release); + } +} + +#[cfg(feature = "lock_api")] +unsafe impl lock_api_crate::RawMutex for SpinMutex<(), R> { + type GuardMarker = lock_api_crate::GuardSend; + + const INIT: Self = Self::new(()); + + fn lock(&self) { + // Prevent guard destructor running + core::mem::forget(Self::lock(self)); + } + + fn try_lock(&self) -> bool { + // Prevent guard destructor running + Self::try_lock(self).map(core::mem::forget).is_some() + } + + unsafe fn unlock(&self) { + self.force_unlock(); + } + + fn is_locked(&self) -> bool { + Self::is_locked(self) + } +} + +#[cfg(test)] +mod tests { + use std::{ + prelude::v1::*, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + mpsc::channel, + }, + thread, + }; + + type SpinMutex = super::SpinMutex; + + #[derive(Eq, PartialEq, Debug)] + struct NonCopy(i32); + + #[test] + fn smoke() { + let m = SpinMutex::<_>::new(()); + drop(m.lock()); + drop(m.lock()); + } + + #[test] + fn lots_and_lots() { + static M: SpinMutex<()> = SpinMutex::<_>::new(()); + static mut CNT: u32 = 0; + const J: u32 = 1000; + const K: u32 = 3; + + fn inc() { + for _ in 0..J { + unsafe { + let _g = M.lock(); + CNT += 1; + } + } + } + + let (tx, rx) = channel(); + let mut ts = Vec::new(); + for _ in 0..K { + let tx2 = tx.clone(); + ts.push(thread::spawn(move || { + inc(); + tx2.send(()).unwrap(); + })); + let tx2 = tx.clone(); + ts.push(thread::spawn(move || { + inc(); + tx2.send(()).unwrap(); + })); + } + + drop(tx); + for _ in 0..2 * K { + rx.recv().unwrap(); + } + assert_eq!(unsafe { CNT }, J * K * 2); + + for t in ts { + t.join().unwrap(); + } + } + + #[test] + fn try_lock() { + let mutex = SpinMutex::<_>::new(42); + + // First lock succeeds + let a = mutex.try_lock(); + assert_eq!(a.as_ref().map(|r| **r), Some(42)); + + // Additional lock fails + let b = mutex.try_lock(); + assert!(b.is_none()); + + // After dropping lock, it succeeds again + ::core::mem::drop(a); + let c = mutex.try_lock(); + assert_eq!(c.as_ref().map(|r| **r), Some(42)); + } + + #[test] + fn test_into_inner() { + let m = SpinMutex::<_>::new(NonCopy(10)); + assert_eq!(m.into_inner(), NonCopy(10)); + } + + #[test] + fn test_into_inner_drop() { + struct Foo(Arc); + impl Drop for Foo { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let num_drops = Arc::new(AtomicUsize::new(0)); + let m = SpinMutex::<_>::new(Foo(num_drops.clone())); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + { + let _inner = m.into_inner(); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + } + assert_eq!(num_drops.load(Ordering::SeqCst), 1); + } + + #[test] + fn test_mutex_arc_nested() { + // Tests nested mutexes and access + // to underlying data. + let arc = Arc::new(SpinMutex::<_>::new(1)); + let arc2 = Arc::new(SpinMutex::<_>::new(arc)); + let (tx, rx) = channel(); + let t = thread::spawn(move || { + let lock = arc2.lock(); + let lock2 = lock.lock(); + assert_eq!(*lock2, 1); + tx.send(()).unwrap(); + }); + rx.recv().unwrap(); + t.join().unwrap(); + } + + #[test] + fn test_mutex_arc_access_in_unwind() { + let arc = Arc::new(SpinMutex::<_>::new(1)); + let arc2 = arc.clone(); + let _ = thread::spawn(move || -> () { + struct Unwinder { + i: Arc>, + } + impl Drop for Unwinder { + fn drop(&mut self) { + *self.i.lock() += 1; + } + } + let _u = Unwinder { i: arc2 }; + panic!(); + }) + .join(); + let lock = arc.lock(); + assert_eq!(*lock, 2); + } + + #[test] + fn test_mutex_unsized() { + let mutex: &SpinMutex<[i32]> = &SpinMutex::<_>::new([1, 2, 3]); + { + let b = &mut *mutex.lock(); + b[0] = 4; + b[2] = 5; + } + let comp: &[i32] = &[4, 2, 5]; + assert_eq!(&*mutex.lock(), comp); + } + + #[test] + fn test_mutex_force_lock() { + let lock = SpinMutex::<_>::new(()); + ::std::mem::forget(lock.lock()); + unsafe { + lock.force_unlock(); + } + assert!(lock.try_lock().is_some()); + } +} diff --git a/components/spin/src/mutex/ticket.rs b/components/spin/src/mutex/ticket.rs new file mode 100644 index 0000000000..534485df8d --- /dev/null +++ b/components/spin/src/mutex/ticket.rs @@ -0,0 +1,551 @@ +//! A ticket-based mutex. +//! +//! Waiting threads take a 'ticket' from the lock in the order they arrive and gain access to the lock when their +//! ticket is next in the queue. Best-case latency is slightly worse than a regular spinning mutex, but worse-case +//! latency is infinitely better. Waiting threads simply need to wait for all threads that come before them in the +//! queue to finish. + +use core::{ + cell::UnsafeCell, + fmt, + marker::PhantomData, + ops::{Deref, DerefMut}, +}; + +use crate::{ + RelaxStrategy, Spin, + atomic::{AtomicUsize, Ordering}, +}; + +/// A spin-based [ticket lock](https://en.wikipedia.org/wiki/Ticket_lock) providing mutually exclusive access to data. +/// +/// A ticket lock is analogous to a queue management system for lock requests. When a thread tries to take a lock, it +/// is assigned a 'ticket'. It then spins until its ticket becomes next in line. When the lock guard is released, the +/// next ticket will be processed. +/// +/// Ticket locks significantly reduce the worse-case performance of locking at the cost of slightly higher average-time +/// overhead. +/// +/// # Example +/// +/// ``` +/// use spin; +/// +/// let lock = spin::mutex::TicketMutex::<_>::new(0); +/// +/// // Modify the data +/// *lock.lock() = 2; +/// +/// // Read the data +/// let answer = *lock.lock(); +/// assert_eq!(answer, 2); +/// ``` +/// +/// # Thread safety example +/// +/// ``` +/// use std::sync::{Arc, Barrier}; +/// +/// use spin; +/// +/// let thread_count = 1000; +/// let spin_mutex = Arc::new(spin::mutex::TicketMutex::<_>::new(0)); +/// +/// // We use a barrier to ensure the readout happens after all writing +/// let barrier = Arc::new(Barrier::new(thread_count + 1)); +/// +/// for _ in (0..thread_count) { +/// let my_barrier = barrier.clone(); +/// let my_lock = spin_mutex.clone(); +/// std::thread::spawn(move || { +/// let mut guard = my_lock.lock(); +/// *guard += 1; +/// +/// // Release the lock to prevent a deadlock +/// drop(guard); +/// my_barrier.wait(); +/// }); +/// } +/// +/// barrier.wait(); +/// +/// let answer = { *spin_mutex.lock() }; +/// assert_eq!(answer, thread_count); +/// ``` +pub struct TicketMutex { + phantom: PhantomData, + next_ticket: AtomicUsize, + next_serving: AtomicUsize, + data: UnsafeCell, +} + +/// A guard that protects some data. +/// +/// When the guard is dropped, the next ticket will be processed. +pub struct TicketMutexGuard<'a, T: ?Sized + 'a> { + next_serving: &'a AtomicUsize, + ticket: usize, + data: &'a mut T, +} + +unsafe impl Sync for TicketMutex {} +unsafe impl Send for TicketMutex {} + +impl TicketMutex { + /// Creates a new [`TicketMutex`] wrapping the supplied data. + /// + /// # Example + /// + /// ``` + /// use spin::mutex::TicketMutex; + /// + /// static MUTEX: TicketMutex<()> = TicketMutex::<_>::new(()); + /// + /// fn demo() { + /// let lock = MUTEX.lock(); + /// // do something with lock + /// drop(lock); + /// } + /// ``` + #[inline(always)] + pub const fn new(data: T) -> Self { + Self { + phantom: PhantomData, + next_ticket: AtomicUsize::new(0), + next_serving: AtomicUsize::new(0), + data: UnsafeCell::new(data), + } + } + + /// Consumes this [`TicketMutex`] and unwraps the underlying data. + /// + /// # Example + /// + /// ``` + /// let lock = spin::mutex::TicketMutex::<_>::new(42); + /// assert_eq!(42, lock.into_inner()); + /// ``` + #[inline(always)] + pub fn into_inner(self) -> T { + self.data.into_inner() + } + /// Returns a mutable pointer to the underying data. + /// + /// This is mostly meant to be used for applications which require manual unlocking, but where + /// storing both the lock and the pointer to the inner data gets inefficient. + /// + /// # Example + /// ``` + /// let lock = spin::mutex::SpinMutex::<_>::new(42); + /// + /// unsafe { + /// core::mem::forget(lock.lock()); + /// + /// assert_eq!(lock.as_mut_ptr().read(), 42); + /// lock.as_mut_ptr().write(58); + /// + /// lock.force_unlock(); + /// } + /// + /// assert_eq!(*lock.lock(), 58); + /// ``` + #[inline(always)] + pub fn as_mut_ptr(&self) -> *mut T { + self.data.get() + } +} + +impl fmt::Debug for TicketMutex { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.try_lock() { + Some(guard) => write!(f, "Mutex {{ data: ") + .and_then(|()| (*guard).fmt(f)) + .and_then(|()| write!(f, " }}")), + None => write!(f, "Mutex {{ }}"), + } + } +} + +impl TicketMutex { + /// Locks the [`TicketMutex`] and returns a guard that permits access to the inner data. + /// + /// The returned data may be dereferenced for data access + /// and the lock will be dropped when the guard falls out of scope. + /// + /// ``` + /// let lock = spin::mutex::TicketMutex::<_>::new(0); + /// { + /// let mut data = lock.lock(); + /// // The lock is now locked and the data can be accessed + /// *data += 1; + /// // The lock is implicitly dropped at the end of the scope + /// } + /// ``` + #[inline(always)] + pub fn lock(&self) -> TicketMutexGuard<'_, T> { + let ticket = self.next_ticket.fetch_add(1, Ordering::Relaxed); + + while self.next_serving.load(Ordering::Acquire) != ticket { + R::relax(); + } + + TicketMutexGuard { + next_serving: &self.next_serving, + ticket, + // Safety + // We know that we are the next ticket to be served, + // so there's no other thread accessing the data. + // + // Every other thread has another ticket number so it's + // definitely stuck in the spin loop above. + data: unsafe { &mut *self.data.get() }, + } + } +} + +impl TicketMutex { + /// Returns `true` if the lock is currently held. + /// + /// # Safety + /// + /// This function provides no synchronization guarantees and so its result should be considered 'out of date' + /// the instant it is called. Do not use it for synchronization purposes. However, it may be useful as a heuristic. + #[inline(always)] + pub fn is_locked(&self) -> bool { + let ticket = self.next_ticket.load(Ordering::Relaxed); + self.next_serving.load(Ordering::Relaxed) != ticket + } + + /// Force unlock this [`TicketMutex`], by serving the next ticket. + /// + /// # Safety + /// + /// This is *extremely* unsafe if the lock is not held by the current + /// thread. However, this can be useful in some instances for exposing the + /// lock to FFI that doesn't know how to deal with RAII. + #[inline(always)] + pub unsafe fn force_unlock(&self) { + self.next_serving.fetch_add(1, Ordering::Release); + } + + /// Try to lock this [`TicketMutex`], returning a lock guard if successful. + /// + /// # Example + /// + /// ``` + /// let lock = spin::mutex::TicketMutex::<_>::new(42); + /// + /// let maybe_guard = lock.try_lock(); + /// assert!(maybe_guard.is_some()); + /// + /// // `maybe_guard` is still held, so the second call fails + /// let maybe_guard2 = lock.try_lock(); + /// assert!(maybe_guard2.is_none()); + /// ``` + #[inline(always)] + pub fn try_lock(&self) -> Option> { + // TODO: Replace with `fetch_update` to avoid manual CAS when upgrading MSRV + let ticket = { + let mut prev = self.next_ticket.load(Ordering::SeqCst); + loop { + if self.next_serving.load(Ordering::Acquire) == prev { + match self.next_ticket.compare_exchange_weak( + prev, + prev + 1, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(x) => break Some(x), + Err(next_prev) => prev = next_prev, + } + } else { + break None; + } + } + }; + + ticket.map(|ticket| TicketMutexGuard { + next_serving: &self.next_serving, + ticket, + // Safety + // We have a ticket that is equal to the next_serving ticket, so we know: + // - that no other thread can have the same ticket id as this thread + // - that we are the next one to be served so we have exclusive access to the data + data: unsafe { &mut *self.data.get() }, + }) + } + + /// Returns a mutable reference to the underlying data. + /// + /// Since this call borrows the [`TicketMutex`] mutably, and a mutable reference is guaranteed to be exclusive in + /// Rust, no actual locking needs to take place -- the mutable borrow statically guarantees no locks exist. As + /// such, this is a 'zero-cost' operation. + /// + /// # Example + /// + /// ``` + /// let mut lock = spin::mutex::TicketMutex::<_>::new(0); + /// *lock.get_mut() = 10; + /// assert_eq!(*lock.lock(), 10); + /// ``` + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + // Safety: + // We know that there are no other references to `self`, + // so it's safe to return a exclusive reference to the data. + unsafe { &mut *self.data.get() } + } +} + +impl Default for TicketMutex { + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl From for TicketMutex { + fn from(data: T) -> Self { + Self::new(data) + } +} + +impl<'a, T: ?Sized> TicketMutexGuard<'a, T> { + /// Leak the lock guard, yielding a mutable reference to the underlying data. + /// + /// Note that this function will permanently lock the original [`TicketMutex`]. + /// + /// ``` + /// let mylock = spin::mutex::TicketMutex::<_>::new(0); + /// + /// let data: &mut i32 = spin::mutex::TicketMutexGuard::leak(mylock.lock()); + /// + /// *data = 1; + /// assert_eq!(*data, 1); + /// ``` + #[inline(always)] + pub fn leak(this: Self) -> &'a mut T { + let data = this.data as *mut _; // Keep it in pointer form temporarily to avoid double-aliasing + core::mem::forget(this); + unsafe { &mut *data } + } +} + +impl<'a, T: ?Sized + fmt::Debug> fmt::Debug for TicketMutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<'a, T: ?Sized + fmt::Display> fmt::Display for TicketMutexGuard<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<'a, T: ?Sized> Deref for TicketMutexGuard<'a, T> { + type Target = T; + fn deref(&self) -> &T { + self.data + } +} + +impl<'a, T: ?Sized> DerefMut for TicketMutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut T { + self.data + } +} + +impl<'a, T: ?Sized> Drop for TicketMutexGuard<'a, T> { + fn drop(&mut self) { + let new_ticket = self.ticket + 1; + self.next_serving.store(new_ticket, Ordering::Release); + } +} + +#[cfg(feature = "lock_api")] +unsafe impl lock_api_crate::RawMutex for TicketMutex<(), R> { + type GuardMarker = lock_api_crate::GuardSend; + + const INIT: Self = Self::new(()); + + fn lock(&self) { + // Prevent guard destructor running + core::mem::forget(Self::lock(self)); + } + + fn try_lock(&self) -> bool { + // Prevent guard destructor running + Self::try_lock(self).map(core::mem::forget).is_some() + } + + unsafe fn unlock(&self) { + self.force_unlock(); + } + + fn is_locked(&self) -> bool { + Self::is_locked(self) + } +} + +#[cfg(test)] +mod tests { + use std::{ + prelude::v1::*, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + mpsc::channel, + }, + thread, + }; + + type TicketMutex = super::TicketMutex; + + #[derive(Eq, PartialEq, Debug)] + struct NonCopy(i32); + + #[test] + fn smoke() { + let m = TicketMutex::<_>::new(()); + drop(m.lock()); + drop(m.lock()); + } + + #[test] + fn lots_and_lots() { + static M: TicketMutex<()> = TicketMutex::<_>::new(()); + static mut CNT: u32 = 0; + const J: u32 = 1000; + const K: u32 = 3; + + fn inc() { + for _ in 0..J { + unsafe { + let _g = M.lock(); + CNT += 1; + } + } + } + + let (tx, rx) = channel(); + for _ in 0..K { + let tx2 = tx.clone(); + thread::spawn(move || { + inc(); + tx2.send(()).unwrap(); + }); + let tx2 = tx.clone(); + thread::spawn(move || { + inc(); + tx2.send(()).unwrap(); + }); + } + + drop(tx); + for _ in 0..2 * K { + rx.recv().unwrap(); + } + assert_eq!(unsafe { CNT }, J * K * 2); + } + + #[test] + fn try_lock() { + let mutex = TicketMutex::<_>::new(42); + + // First lock succeeds + let a = mutex.try_lock(); + assert_eq!(a.as_ref().map(|r| **r), Some(42)); + + // Additional lock fails + let b = mutex.try_lock(); + assert!(b.is_none()); + + // After dropping lock, it succeeds again + ::core::mem::drop(a); + let c = mutex.try_lock(); + assert_eq!(c.as_ref().map(|r| **r), Some(42)); + } + + #[test] + fn test_into_inner() { + let m = TicketMutex::<_>::new(NonCopy(10)); + assert_eq!(m.into_inner(), NonCopy(10)); + } + + #[test] + fn test_into_inner_drop() { + struct Foo(Arc); + impl Drop for Foo { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let num_drops = Arc::new(AtomicUsize::new(0)); + let m = TicketMutex::<_>::new(Foo(num_drops.clone())); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + { + let _inner = m.into_inner(); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + } + assert_eq!(num_drops.load(Ordering::SeqCst), 1); + } + + #[test] + fn test_mutex_arc_nested() { + // Tests nested mutexes and access + // to underlying data. + let arc = Arc::new(TicketMutex::<_>::new(1)); + let arc2 = Arc::new(TicketMutex::<_>::new(arc)); + let (tx, rx) = channel(); + let _t = thread::spawn(move || { + let lock = arc2.lock(); + let lock2 = lock.lock(); + assert_eq!(*lock2, 1); + tx.send(()).unwrap(); + }); + rx.recv().unwrap(); + } + + #[test] + fn test_mutex_arc_access_in_unwind() { + let arc = Arc::new(TicketMutex::<_>::new(1)); + let arc2 = arc.clone(); + let _ = thread::spawn(move || -> () { + struct Unwinder { + i: Arc>, + } + impl Drop for Unwinder { + fn drop(&mut self) { + *self.i.lock() += 1; + } + } + let _u = Unwinder { i: arc2 }; + panic!(); + }) + .join(); + let lock = arc.lock(); + assert_eq!(*lock, 2); + } + + #[test] + fn test_mutex_unsized() { + let mutex: &TicketMutex<[i32]> = &TicketMutex::<_>::new([1, 2, 3]); + { + let b = &mut *mutex.lock(); + b[0] = 4; + b[2] = 5; + } + let comp: &[i32] = &[4, 2, 5]; + assert_eq!(&*mutex.lock(), comp); + } + + #[test] + fn is_locked() { + let mutex = TicketMutex::<_>::new(()); + assert!(!mutex.is_locked()); + let lock = mutex.lock(); + assert!(mutex.is_locked()); + drop(lock); + assert!(!mutex.is_locked()); + } +} diff --git a/components/spin/src/once.rs b/components/spin/src/once.rs new file mode 100644 index 0000000000..4c54b7e12d --- /dev/null +++ b/components/spin/src/once.rs @@ -0,0 +1,795 @@ +//! Synchronization primitives for one-time evaluation. + +use core::{cell::UnsafeCell, fmt, marker::PhantomData, mem::MaybeUninit}; + +use crate::{ + RelaxStrategy, Spin, + atomic::{AtomicU8, Ordering}, +}; + +/// A primitive that provides lazy one-time initialization. +/// +/// Unlike its `std::sync` equivalent, this is generalized such that the closure returns a +/// value to be stored by the [`Once`] (`std::sync::Once` can be trivially emulated with +/// `Once`). +/// +/// Because [`Once::new`] is `const`, this primitive may be used to safely initialize statics. +/// +/// # Examples +/// +/// ``` +/// use spin; +/// +/// static START: spin::Once = spin::Once::new(); +/// +/// START.call_once(|| { +/// // run initialization here +/// }); +/// ``` +pub struct Once { + phantom: PhantomData, + status: AtomicStatus, + data: UnsafeCell>, +} + +impl Default for Once { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for Once { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut d = f.debug_tuple("Once"); + let d = if let Some(x) = self.get() { + d.field(&x) + } else { + d.field(&format_args!("")) + }; + d.finish() + } +} + +// Same unsafe impls as `std::sync::RwLock`, because this also allows for +// concurrent reads. +unsafe impl Sync for Once {} +unsafe impl Send for Once {} + +mod status { + use super::*; + + // SAFETY: This structure has an invariant, namely that the inner atomic u8 must *always* have + // a value for which there exists a valid Status. This means that users of this API must only + // be allowed to load and store `Status`es. + #[repr(transparent)] + pub struct AtomicStatus(AtomicU8); + + // Four states that a Once can be in, encoded into the lower bits of `status` in + // the Once structure. + #[repr(u8)] + #[derive(Clone, Copy, Debug, PartialEq)] + pub enum Status { + Incomplete = 0x00, + Running = 0x01, + Complete = 0x02, + Panicked = 0x03, + } + impl Status { + // Construct a status from an inner u8 integer. + // + // # Safety + // + // For this to be safe, the inner number must have a valid corresponding enum variant. + unsafe fn new_unchecked(inner: u8) -> Self { + core::mem::transmute(inner) + } + } + + impl AtomicStatus { + #[inline(always)] + pub const fn new(status: Status) -> Self { + // SAFETY: We got the value directly from status, so transmuting back is fine. + Self(AtomicU8::new(status as u8)) + } + #[inline(always)] + pub fn load(&self, ordering: Ordering) -> Status { + // SAFETY: We know that the inner integer must have been constructed from a Status in + // the first place. + unsafe { Status::new_unchecked(self.0.load(ordering)) } + } + #[inline(always)] + pub fn store(&self, status: Status, ordering: Ordering) { + // SAFETY: While not directly unsafe, this is safe because the value was retrieved from + // a status, thus making transmutation safe. + self.0.store(status as u8, ordering); + } + #[inline(always)] + pub fn compare_exchange( + &self, + old: Status, + new: Status, + success: Ordering, + failure: Ordering, + ) -> Result { + match self + .0 + .compare_exchange(old as u8, new as u8, success, failure) + { + // SAFETY: A compare exchange will always return a value that was later stored into + // the atomic u8, but due to the invariant that it must be a valid Status, we know + // that both Ok(_) and Err(_) will be safely transmutable. + Ok(ok) => Ok(unsafe { Status::new_unchecked(ok) }), + Err(err) => Err(unsafe { Status::new_unchecked(err) }), + } + } + #[inline(always)] + pub fn get_mut(&mut self) -> &mut Status { + // SAFETY: Since we know that the u8 inside must be a valid Status, we can safely cast + // it to a &mut Status. + unsafe { &mut *((self.0.get_mut() as *mut u8).cast::()) } + } + } +} +use self::status::{AtomicStatus, Status}; + +impl Once { + /// Performs an initialization routine once and only once. The given closure + /// will be executed if this is the first time `call_once` has been called, + /// and otherwise the routine will *not* be invoked. + /// + /// This method will block the calling thread if another initialization + /// routine is currently running. + /// + /// When this function returns, it is guaranteed that some initialization + /// has run and completed (it may not be the closure specified). The + /// returned pointer will point to the result from the closure that was + /// run. + /// + /// # Panics + /// + /// This function will panic if the [`Once`] previously panicked while attempting + /// to initialize. This is similar to the poisoning behaviour of `std::sync`'s + /// primitives. + /// + /// # Examples + /// + /// ``` + /// use spin; + /// + /// static INIT: spin::Once = spin::Once::new(); + /// + /// fn get_cached_val() -> usize { + /// *INIT.call_once(expensive_computation) + /// } + /// + /// fn expensive_computation() -> usize { + /// // ... + /// # 2 + /// } + /// ``` + pub fn call_once T>(&self, f: F) -> &T { + match self.try_call_once(|| Ok::(f())) { + Ok(x) => x, + Err(void) => match void {}, + } + } + + /// This method is similar to `call_once`, but allows the given closure to + /// fail, and lets the `Once` in a uninitialized state if it does. + /// + /// This method will block the calling thread if another initialization + /// routine is currently running. + /// + /// When this function returns without error, it is guaranteed that some + /// initialization has run and completed (it may not be the closure + /// specified). The returned reference will point to the result from the + /// closure that was run. + /// + /// # Panics + /// + /// This function will panic if the [`Once`] previously panicked while attempting + /// to initialize. This is similar to the poisoning behaviour of `std::sync`'s + /// primitives. + /// + /// # Examples + /// + /// ``` + /// use spin; + /// + /// static INIT: spin::Once = spin::Once::new(); + /// + /// fn get_cached_val() -> Result { + /// INIT.try_call_once(expensive_fallible_computation) + /// .map(|x| *x) + /// } + /// + /// fn expensive_fallible_computation() -> Result { + /// // ... + /// # Ok(2) + /// } + /// ``` + pub fn try_call_once Result, E>(&self, f: F) -> Result<&T, E> { + if let Some(value) = self.get() { + Ok(value) + } else { + self.try_call_once_slow(f) + } + } + + #[cold] + fn try_call_once_slow Result, E>(&self, f: F) -> Result<&T, E> { + loop { + let xchg = self.status.compare_exchange( + Status::Incomplete, + Status::Running, + Ordering::Acquire, + Ordering::Acquire, + ); + + match xchg { + Ok(_must_be_state_incomplete) => { + // Impl is defined after the match for readability + } + Err(Status::Panicked) => panic!("Once panicked"), + Err(Status::Running) => match self.poll() { + Some(v) => return Ok(v), + None => continue, + }, + Err(Status::Complete) => { + return Ok(unsafe { + // SAFETY: The status is Complete + self.force_get() + }); + } + Err(Status::Incomplete) => { + // The compare_exchange failed, so this shouldn't ever be reached, + // however if we decide to switch to compare_exchange_weak it will + // be safer to leave this here than hit an unreachable + continue; + } + } + + // The compare-exchange succeeded, so we shall initialize it. + + // We use a guard (Finish) to catch panics caused by builder + let finish = Finish { + status: &self.status, + }; + let val = match f() { + Ok(val) => val, + Err(err) => { + // If an error occurs, clean up everything and leave. + core::mem::forget(finish); + self.status.store(Status::Incomplete, Ordering::Release); + return Err(err); + } + }; + unsafe { + // SAFETY: + // `UnsafeCell`/deref: currently the only accessor, mutably + // and immutably by cas exclusion. + // `write`: pointer comes from `MaybeUninit`. + (*self.data.get()).as_mut_ptr().write(val); + }; + // If there were to be a panic with unwind enabled, the code would + // short-circuit and never reach the point where it writes the inner data. + // The destructor for Finish will run, and poison the Once to ensure that other + // threads accessing it do not exhibit unwanted behavior, if there were to be + // any inconsistency in data structures caused by the panicking thread. + // + // However, f() is expected in the general case not to panic. In that case, we + // simply forget the guard, bypassing its destructor. We could theoretically + // clear a flag instead, but this eliminates the call to the destructor at + // compile time, and unconditionally poisons during an eventual panic, if + // unwinding is enabled. + core::mem::forget(finish); + + // SAFETY: Release is required here, so that all memory accesses done in the + // closure when initializing, become visible to other threads that perform Acquire + // loads. + // + // And, we also know that the changes this thread has done will not magically + // disappear from our cache, so it does not need to be AcqRel. + self.status.store(Status::Complete, Ordering::Release); + + // This next line is mainly an optimization. + return unsafe { Ok(self.force_get()) }; + } + } + + /// Spins until the [`Once`] contains a value. + /// + /// Note that in releases prior to `0.7`, this function had the behaviour of [`Once::poll`]. + /// + /// # Panics + /// + /// This function will panic if the [`Once`] previously panicked while attempting + /// to initialize. This is similar to the poisoning behaviour of `std::sync`'s + /// primitives. + pub fn wait(&self) -> &T { + loop { + match self.poll() { + Some(x) => break x, + None => R::relax(), + } + } + } + + /// Like [`Once::get`], but will spin if the [`Once`] is in the process of being + /// initialized. If initialization has not even begun, `None` will be returned. + /// + /// Note that in releases prior to `0.7`, this function was named `wait`. + /// + /// # Panics + /// + /// This function will panic if the [`Once`] previously panicked while attempting + /// to initialize. This is similar to the poisoning behaviour of `std::sync`'s + /// primitives. + pub fn poll(&self) -> Option<&T> { + loop { + // SAFETY: Acquire is safe here, because if the status is COMPLETE, then we want to make + // sure that all memory accessed done while initializing that value, are visible when + // we return a reference to the inner data after this load. + match self.status.load(Ordering::Acquire) { + Status::Incomplete => return None, + Status::Running => R::relax(), // We spin + Status::Complete => return Some(unsafe { self.force_get() }), + Status::Panicked => panic!("Once previously poisoned by a panicked"), + } + } + } +} + +impl Once { + /// Initialization constant of [`Once`]. + #[allow(clippy::declare_interior_mutable_const)] + pub const INIT: Self = Self { + phantom: PhantomData, + status: AtomicStatus::new(Status::Incomplete), + data: UnsafeCell::new(MaybeUninit::uninit()), + }; + + /// Creates a new [`Once`]. + pub const fn new() -> Self { + Self::INIT + } + + /// Creates a new initialized [`Once`]. + pub const fn initialized(data: T) -> Self { + Self { + phantom: PhantomData, + status: AtomicStatus::new(Status::Complete), + data: UnsafeCell::new(MaybeUninit::new(data)), + } + } + + /// Retrieve a pointer to the inner data. + /// + /// While this method itself is safe, accessing the pointer before the [`Once`] has been + /// initialized is UB, unless this method has already been written to from a pointer coming + /// from this method. + pub fn as_mut_ptr(&self) -> *mut T { + // SAFETY: + // * MaybeUninit always has exactly the same layout as T + self.data.get().cast::() + } + + /// Get a reference to the initialized instance. Must only be called once COMPLETE. + unsafe fn force_get(&self) -> &T { + // SAFETY: + // * `UnsafeCell`/inner deref: data never changes again + // * `MaybeUninit`/outer deref: data was initialized + &*(*self.data.get()).as_ptr() + } + + /// Get a reference to the initialized instance. Must only be called once COMPLETE. + unsafe fn force_get_mut(&mut self) -> &mut T { + // SAFETY: + // * `UnsafeCell`/inner deref: data never changes again + // * `MaybeUninit`/outer deref: data was initialized + &mut *(*self.data.get()).as_mut_ptr() + } + + /// Get a reference to the initialized instance. Must only be called once COMPLETE. + unsafe fn force_into_inner(self) -> T { + // SAFETY: + // * `UnsafeCell`/inner deref: data never changes again + // * `MaybeUninit`/outer deref: data was initialized + (*self.data.get()).as_ptr().read() + } + + /// Returns a reference to the inner value if the [`Once`] has been initialized. + pub fn get(&self) -> Option<&T> { + // SAFETY: Just as with `poll`, Acquire is safe here because we want to be able to see the + // nonatomic stores done when initializing, once we have loaded and checked the status. + match self.status.load(Ordering::Acquire) { + Status::Complete => Some(unsafe { self.force_get() }), + _ => None, + } + } + + /// Returns a reference to the inner value on the unchecked assumption that the [`Once`] has been initialized. + /// + /// # Safety + /// + /// This is *extremely* unsafe if the `Once` has not already been initialized because a reference to uninitialized + /// memory will be returned, immediately triggering undefined behaviour (even if the reference goes unused). + /// However, this can be useful in some instances for exposing the `Once` to FFI or when the overhead of atomically + /// checking initialization is unacceptable and the `Once` has already been initialized. + pub unsafe fn get_unchecked(&self) -> &T { + debug_assert_eq!( + self.status.load(Ordering::SeqCst), + Status::Complete, + "Attempted to access an uninitialized Once. If this was run without debug checks, \ + this would be undefined behaviour. This is a serious bug and you must fix it.", + ); + self.force_get() + } + + /// Returns a mutable reference to the inner value if the [`Once`] has been initialized. + /// + /// Because this method requires a mutable reference to the [`Once`], no synchronization + /// overhead is required to access the inner value. In effect, it is zero-cost. + pub fn get_mut(&mut self) -> Option<&mut T> { + match *self.status.get_mut() { + Status::Complete => Some(unsafe { self.force_get_mut() }), + _ => None, + } + } + + /// Returns a mutable reference to the inner value + /// + /// # Safety + /// + /// This is *extremely* unsafe if the `Once` has not already been initialized because a reference to uninitialized + /// memory will be returned, immediately triggering undefined behaviour (even if the reference goes unused). + /// However, this can be useful in some instances for exposing the `Once` to FFI or when the overhead of atomically + /// checking initialization is unacceptable and the `Once` has already been initialized. + pub unsafe fn get_mut_unchecked(&mut self) -> &mut T { + debug_assert_eq!( + self.status.load(Ordering::SeqCst), + Status::Complete, + "Attempted to access an unintialized Once. If this was to run without debug checks, \ + this would be undefined behavior. This is a serious bug and you must fix it.", + ); + self.force_get_mut() + } + + /// Returns a the inner value if the [`Once`] has been initialized. + /// + /// Because this method requires ownership of the [`Once`], no synchronization overhead + /// is required to access the inner value. In effect, it is zero-cost. + pub fn try_into_inner(mut self) -> Option { + match *self.status.get_mut() { + Status::Complete => Some(unsafe { self.force_into_inner() }), + _ => None, + } + } + + /// Returns a the inner value if the [`Once`] has been initialized. + /// # Safety + /// + /// This is *extremely* unsafe if the `Once` has not already been initialized because a reference to uninitialized + /// memory will be returned, immediately triggering undefined behaviour (even if the reference goes unused) + /// This can be useful, if `Once` has already been initialized, and you want to bypass an + /// option check. + pub unsafe fn into_inner_unchecked(self) -> T { + debug_assert_eq!( + self.status.load(Ordering::SeqCst), + Status::Complete, + "Attempted to access an unintialized Once. If this was to run without debug checks, \ + this would be undefined behavior. This is a serious bug and you must fix it.", + ); + self.force_into_inner() + } + + /// Checks whether the value has been initialized. + /// + /// This is done using [`Acquire`](core::sync::atomic::Ordering::Acquire) ordering, and + /// therefore it is safe to access the value directly via + /// [`get_unchecked`](Self::get_unchecked) if this returns true. + pub fn is_completed(&self) -> bool { + // TODO: Add a similar variant for Relaxed? + self.status.load(Ordering::Acquire) == Status::Complete + } +} + +impl From for Once { + fn from(data: T) -> Self { + Self::initialized(data) + } +} + +impl Drop for Once { + fn drop(&mut self) { + // No need to do any atomic access here, we have &mut! + if *self.status.get_mut() == Status::Complete { + unsafe { + // TODO: Use MaybeUninit::assume_init_drop once stabilised + core::ptr::drop_in_place((*self.data.get()).as_mut_ptr()); + } + } + } +} + +struct Finish<'a> { + status: &'a AtomicStatus, +} + +impl<'a> Drop for Finish<'a> { + fn drop(&mut self) { + // While using Relaxed here would most likely not be an issue, we use SeqCst anyway. + // This is mainly because panics are not meant to be fast at all, but also because if + // there were to be a compiler bug which reorders accesses within the same thread, + // where it should not, we want to be sure that the panic really is handled, and does + // not cause additional problems. SeqCst will therefore help guarding against such + // bugs. + self.status.store(Status::Panicked, Ordering::SeqCst); + } +} + +#[cfg(test)] +mod tests { + use std::{ + prelude::v1::*, + sync::{Arc, atomic::AtomicU32, mpsc::channel}, + thread, + }; + + use super::*; + + #[test] + fn smoke_once() { + static O: Once = Once::new(); + let mut a = 0; + O.call_once(|| a += 1); + assert_eq!(a, 1); + O.call_once(|| a += 1); + assert_eq!(a, 1); + } + + #[test] + fn smoke_once_value() { + static O: Once = Once::new(); + let a = O.call_once(|| 1); + assert_eq!(*a, 1); + let b = O.call_once(|| 2); + assert_eq!(*b, 1); + } + + #[test] + fn stampede_once() { + static O: Once = Once::new(); + static mut RUN: bool = false; + + let (tx, rx) = channel(); + let mut ts = Vec::new(); + for _ in 0..10 { + let tx = tx.clone(); + ts.push(thread::spawn(move || { + for _ in 0..4 { + thread::yield_now() + } + unsafe { + O.call_once(|| { + assert!(!RUN); + RUN = true; + }); + assert!(RUN); + } + tx.send(()).unwrap(); + })); + } + + unsafe { + O.call_once(|| { + assert!(!RUN); + RUN = true; + }); + assert!(RUN); + } + + for _ in 0..10 { + rx.recv().unwrap(); + } + + for t in ts { + t.join().unwrap(); + } + } + + #[test] + fn get() { + static INIT: Once = Once::new(); + + assert!(INIT.get().is_none()); + INIT.call_once(|| 2); + assert_eq!(INIT.get().map(|r| *r), Some(2)); + } + + #[test] + fn get_no_wait() { + static INIT: Once = Once::new(); + + assert!(INIT.get().is_none()); + let t = thread::spawn(move || { + INIT.call_once(|| { + thread::sleep(std::time::Duration::from_secs(3)); + 42 + }); + }); + assert!(INIT.get().is_none()); + + t.join().unwrap(); + } + + #[test] + fn poll() { + static INIT: Once = Once::new(); + + assert!(INIT.poll().is_none()); + INIT.call_once(|| 3); + assert_eq!(INIT.poll().map(|r| *r), Some(3)); + } + + #[test] + fn wait() { + static INIT: Once = Once::new(); + + let t = std::thread::spawn(|| { + assert_eq!(*INIT.wait(), 3); + assert!(INIT.is_completed()); + }); + + for _ in 0..4 { + thread::yield_now() + } + + assert!(INIT.poll().is_none()); + INIT.call_once(|| 3); + + t.join().unwrap(); + } + + #[test] + fn panic() { + use std::panic; + + static INIT: Once = Once::new(); + + // poison the once + let t = panic::catch_unwind(|| { + INIT.call_once(|| panic!()); + }); + assert!(t.is_err()); + + // poisoning propagates + let t = panic::catch_unwind(|| { + INIT.call_once(|| {}); + }); + assert!(t.is_err()); + } + + #[test] + fn init_constant() { + static O: Once = Once::INIT; + let mut a = 0; + O.call_once(|| a += 1); + assert_eq!(a, 1); + O.call_once(|| a += 1); + assert_eq!(a, 1); + } + + static mut CALLED: bool = false; + + struct DropTest {} + + impl Drop for DropTest { + fn drop(&mut self) { + unsafe { + CALLED = true; + } + } + } + + #[test] + fn try_call_once_err() { + let once = Once::<_, Spin>::new(); + let shared = Arc::new((once, AtomicU32::new(0))); + + let (tx, rx) = channel(); + + let t0 = { + let shared = shared.clone(); + thread::spawn(move || { + let (once, called) = &*shared; + + once.try_call_once(|| { + called.fetch_add(1, Ordering::AcqRel); + tx.send(()).unwrap(); + thread::sleep(std::time::Duration::from_millis(50)); + Err(()) + }) + .ok(); + }) + }; + + let t1 = { + let shared = shared.clone(); + thread::spawn(move || { + rx.recv().unwrap(); + let (once, called) = &*shared; + assert_eq!( + called.load(Ordering::Acquire), + 1, + "leader thread did not run first" + ); + + once.call_once(|| { + called.fetch_add(1, Ordering::AcqRel); + }); + }) + }; + + t0.join().unwrap(); + t1.join().unwrap(); + + assert_eq!(shared.1.load(Ordering::Acquire), 2); + } + + // This is sort of two test cases, but if we write them as separate test methods + // they can be executed concurrently and then fail some small fraction of the + // time. + #[test] + fn drop_occurs_and_skip_uninit_drop() { + unsafe { + CALLED = false; + } + + { + let once = Once::<_>::new(); + once.call_once(|| DropTest {}); + } + + assert!(unsafe { CALLED }); + // Now test that we skip drops for the uninitialized case. + unsafe { + CALLED = false; + } + + let once = Once::::new(); + drop(once); + + assert!(unsafe { !CALLED }); + } + + #[test] + fn call_once_test() { + for _ in 0..20 { + use std::{ + sync::{Arc, atomic::AtomicUsize}, + time::Duration, + }; + let share = Arc::new(AtomicUsize::new(0)); + let once = Arc::new(Once::<_, Spin>::new()); + let mut hs = Vec::new(); + for _ in 0..8 { + let h = thread::spawn({ + let share = share.clone(); + let once = once.clone(); + move || { + thread::sleep(Duration::from_millis(10)); + once.call_once(|| { + share.fetch_add(1, Ordering::SeqCst); + }); + } + }); + hs.push(h); + } + for h in hs { + h.join().unwrap(); + } + assert_eq!(1, share.load(Ordering::SeqCst)); + } + } +} diff --git a/components/spin/src/relax.rs b/components/spin/src/relax.rs new file mode 100644 index 0000000000..8842f80bcd --- /dev/null +++ b/components/spin/src/relax.rs @@ -0,0 +1,61 @@ +//! Strategies that determine the behaviour of locks when encountering contention. + +/// A trait implemented by spinning relax strategies. +pub trait RelaxStrategy { + /// Perform the relaxing operation during a period of contention. + fn relax(); +} + +/// A strategy that rapidly spins while informing the CPU that it should power down non-essential components via +/// [`core::hint::spin_loop`]. +/// +/// Note that spinning is a 'dumb' strategy and most schedulers cannot correctly differentiate it from useful work, +/// thereby misallocating even more CPU time to the spinning process. This is known as +/// ['priority inversion'](https://matklad.github.io/2020/01/02/spinlocks-considered-harmful.html). +/// +/// If you see signs that priority inversion is occurring, consider switching to [`Yield`] or, even better, not using a +/// spinlock at all and opting for a proper scheduler-aware lock. Remember also that different targets, operating +/// systems, schedulers, and even the same scheduler with different workloads will exhibit different behaviour. Just +/// because priority inversion isn't occurring in your tests does not mean that it will not occur. Use a scheduler- +/// aware lock if at all possible. +pub struct Spin; + +impl RelaxStrategy for Spin { + #[inline(always)] + fn relax() { + // Use the deprecated spin_loop_hint() to ensure that we don't get + // a higher MSRV than we need to. + #[allow(deprecated)] + core::sync::atomic::spin_loop_hint(); + } +} + +/// A strategy that yields the current time slice to the scheduler in favour of other threads or processes. +/// +/// This is generally used as a strategy for minimising power consumption and priority inversion on targets that have a +/// standard library available. Note that such targets have scheduler-integrated concurrency primitives available, and +/// you should generally use these instead, except in rare circumstances. +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +pub struct Yield; + +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +impl RelaxStrategy for Yield { + #[inline(always)] + fn relax() { + std::thread::yield_now(); + } +} + +/// A strategy that rapidly spins, without telling the CPU to do any powering down. +/// +/// You almost certainly do not want to use this. Use [`Spin`] instead. It exists for completeness and for targets +/// that, for some reason, miscompile or do not support spin hint intrinsics despite attempting to generate code for +/// them (i.e: this is a workaround for possible compiler bugs). +pub struct Loop; + +impl RelaxStrategy for Loop { + #[inline(always)] + fn relax() {} +} diff --git a/components/spin/src/rwlock.rs b/components/spin/src/rwlock.rs new file mode 100644 index 0000000000..80504c257e --- /dev/null +++ b/components/spin/src/rwlock.rs @@ -0,0 +1,1190 @@ +//! A lock that provides data access to either one writer or many readers. + +use core::{ + cell::UnsafeCell, + fmt, + marker::PhantomData, + mem, + mem::ManuallyDrop, + ops::{Deref, DerefMut}, +}; + +use crate::{ + RelaxStrategy, Spin, + atomic::{AtomicUsize, Ordering}, +}; + +/// A lock that provides data access to either one writer or many readers. +/// +/// This lock behaves in a similar manner to its namesake `std::sync::RwLock` but uses +/// spinning for synchronisation instead. Unlike its namespace, this lock does not +/// track lock poisoning. +/// +/// This type of lock allows a number of readers or at most one writer at any +/// point in time. The write portion of this lock typically allows modification +/// of the underlying data (exclusive access) and the read portion of this lock +/// typically allows for read-only access (shared access). +/// +/// The type parameter `T` represents the data that this lock protects. It is +/// required that `T` satisfies `Send` to be shared across tasks and `Sync` to +/// allow concurrent access through readers. The RAII guards returned from the +/// locking methods implement `Deref` (and `DerefMut` for the `write` methods) +/// to allow access to the contained of the lock. +/// +/// An [`RwLockUpgradableGuard`](RwLockUpgradableGuard) can be upgraded to a +/// writable guard through the [`RwLockUpgradableGuard::upgrade`](RwLockUpgradableGuard::upgrade) +/// [`RwLockUpgradableGuard::try_upgrade`](RwLockUpgradableGuard::try_upgrade) functions. +/// Writable or upgradeable guards can be downgraded through their respective `downgrade` +/// functions. +/// +/// Based on Facebook's +/// [`folly/RWSpinLock.h`](https://github.com/facebook/folly/blob/a0394d84f2d5c3e50ebfd0566f9d3acb52cfab5a/folly/synchronization/RWSpinLock.h). +/// This implementation is unfair to writers - if the lock always has readers, then no writers will +/// ever get a chance. Using an upgradeable lock guard can *somewhat* alleviate this issue as no +/// new readers are allowed when an upgradeable guard is held, but upgradeable guards can be taken +/// when there are existing readers. However if the lock is that highly contended and writes are +/// crucial then this implementation may be a poor choice. +/// +/// # Examples +/// +/// ``` +/// use spin; +/// +/// let lock = spin::RwLock::new(5); +/// +/// // many reader locks can be held at once +/// { +/// let r1 = lock.read(); +/// let r2 = lock.read(); +/// assert_eq!(*r1, 5); +/// assert_eq!(*r2, 5); +/// } // read locks are dropped at this point +/// +/// // only one write lock may be held, however +/// { +/// let mut w = lock.write(); +/// *w += 1; +/// assert_eq!(*w, 6); +/// } // write lock is dropped here +/// ``` +pub struct RwLock { + phantom: PhantomData, + lock: AtomicUsize, + data: UnsafeCell, +} + +const READER: usize = 1 << 2; +const UPGRADED: usize = 1 << 1; +const WRITER: usize = 1; + +/// A guard that provides immutable data access. +/// +/// When the guard falls out of scope it will decrement the read count, +/// potentially releasing the lock. +pub struct RwLockReadGuard<'a, T: 'a + ?Sized> { + lock: &'a AtomicUsize, + data: *const T, +} + +/// A guard that provides mutable data access. +/// +/// When the guard falls out of scope it will release the lock. +pub struct RwLockWriteGuard<'a, T: 'a + ?Sized, R = Spin> { + phantom: PhantomData, + inner: &'a RwLock, + data: *mut T, +} + +/// A guard that provides immutable data access but can be upgraded to [`RwLockWriteGuard`]. +/// +/// No writers or other upgradeable guards can exist while this is in scope. New reader +/// creation is prevented (to alleviate writer starvation) but there may be existing readers +/// when the lock is acquired. +/// +/// When the guard falls out of scope it will release the lock. +pub struct RwLockUpgradableGuard<'a, T: 'a + ?Sized, R = Spin> { + phantom: PhantomData, + inner: &'a RwLock, + data: *const T, +} + +// Same unsafe impls as `std::sync::RwLock` +unsafe impl Send for RwLock {} +unsafe impl Sync for RwLock {} + +unsafe impl Send for RwLockWriteGuard<'_, T, R> {} +unsafe impl Sync for RwLockWriteGuard<'_, T, R> {} + +unsafe impl Send for RwLockReadGuard<'_, T> {} +unsafe impl Sync for RwLockReadGuard<'_, T> {} + +unsafe impl Send for RwLockUpgradableGuard<'_, T, R> {} +unsafe impl Sync for RwLockUpgradableGuard<'_, T, R> {} + +impl RwLock { + /// Creates a new spinlock wrapping the supplied data. + /// + /// May be used statically: + /// + /// ``` + /// use spin; + /// + /// static RW_LOCK: spin::RwLock<()> = spin::RwLock::new(()); + /// + /// fn demo() { + /// let lock = RW_LOCK.read(); + /// // do something with lock + /// drop(lock); + /// } + /// ``` + #[inline] + pub const fn new(data: T) -> Self { + RwLock { + phantom: PhantomData, + lock: AtomicUsize::new(0), + data: UnsafeCell::new(data), + } + } + + /// Consumes this `RwLock`, returning the underlying data. + #[inline] + pub fn into_inner(self) -> T { + // We know statically that there are no outstanding references to + // `self` so there's no need to lock. + let RwLock { data, .. } = self; + data.into_inner() + } + /// Returns a mutable pointer to the underying data. + /// + /// This is mostly meant to be used for applications which require manual unlocking, but where + /// storing both the lock and the pointer to the inner data gets inefficient. + /// + /// While this is safe, writing to the data is undefined behavior unless the current thread has + /// acquired a write lock, and reading requires either a read or write lock. + /// + /// # Example + /// ``` + /// let lock = spin::RwLock::new(42); + /// + /// unsafe { + /// core::mem::forget(lock.write()); + /// + /// assert_eq!(lock.as_mut_ptr().read(), 42); + /// lock.as_mut_ptr().write(58); + /// + /// lock.force_write_unlock(); + /// } + /// + /// assert_eq!(*lock.read(), 58); + /// ``` + #[inline(always)] + pub fn as_mut_ptr(&self) -> *mut T { + self.data.get() + } +} + +impl RwLock { + /// Locks this rwlock with shared read access, blocking the current thread + /// until it can be acquired. + /// + /// The calling thread will be blocked until there are no more writers which + /// hold the lock. There may be other readers currently inside the lock when + /// this method returns. This method does not provide any guarantees with + /// respect to the ordering of whether contentious readers or writers will + /// acquire the lock first. + /// + /// Returns an RAII guard which will release this thread's shared access + /// once it is dropped. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// { + /// let mut data = mylock.read(); + /// // The lock is now locked and the data can be read + /// println!("{}", *data); + /// // The lock is dropped + /// } + /// ``` + #[inline] + pub fn read(&self) -> RwLockReadGuard<'_, T> { + loop { + match self.try_read() { + Some(guard) => return guard, + None => R::relax(), + } + } + } + + /// Lock this rwlock with exclusive write access, blocking the current + /// thread until it can be acquired. + /// + /// This function will not return while other writers or other readers + /// currently have access to the lock. + /// + /// Returns an RAII guard which will drop the write access of this rwlock + /// when dropped. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// { + /// let mut data = mylock.write(); + /// // The lock is now locked and the data can be written + /// *data += 1; + /// // The lock is dropped + /// } + /// ``` + #[inline] + pub fn write(&self) -> RwLockWriteGuard<'_, T, R> { + loop { + match self.try_write_internal(false) { + Some(guard) => return guard, + None => R::relax(), + } + } + } + + /// Obtain a readable lock guard that can later be upgraded to a writable lock guard. + /// Upgrades can be done through the [`RwLockUpgradableGuard::upgrade`](RwLockUpgradableGuard::upgrade) method. + #[inline] + pub fn upgradeable_read(&self) -> RwLockUpgradableGuard<'_, T, R> { + loop { + match self.try_upgradeable_read() { + Some(guard) => return guard, + None => R::relax(), + } + } + } +} + +impl RwLock { + // Acquire a read lock, returning the new lock value. + fn acquire_reader(&self) -> usize { + // An arbitrary cap that allows us to catch overflows long before they happen + const MAX_READERS: usize = usize::MAX / READER / 2; + + let value = self.lock.fetch_add(READER, Ordering::Acquire); + + if value > MAX_READERS * READER { + self.lock.fetch_sub(READER, Ordering::Relaxed); + panic!("Too many lock readers, cannot safely proceed"); + } else { + value + } + } + + /// Attempt to acquire this lock with shared read access. + /// + /// This function will never block and will return immediately if `read` + /// would otherwise succeed. Returns `Some` of an RAII guard which will + /// release the shared access of this thread when dropped, or `None` if the + /// access could not be granted. This method does not provide any + /// guarantees with respect to the ordering of whether contentious readers + /// or writers will acquire the lock first. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// { + /// match mylock.try_read() { + /// Some(data) => { + /// // The lock is now locked and the data can be read + /// println!("{}", *data); + /// // The lock is dropped + /// } + /// None => (), // no cigar + /// }; + /// } + /// ``` + #[inline] + pub fn try_read(&self) -> Option> { + let value = self.acquire_reader(); + + // We check the UPGRADED bit here so that new readers are prevented when an UPGRADED lock is held. + // This helps reduce writer starvation. + if value & (WRITER | UPGRADED) != 0 { + // Lock is taken, undo. + self.lock.fetch_sub(READER, Ordering::Release); + None + } else { + Some(RwLockReadGuard { + lock: &self.lock, + data: unsafe { &*self.data.get() }, + }) + } + } + + /// Return the number of readers that currently hold the lock (including upgradable readers). + /// + /// # Safety + /// + /// This function provides no synchronization guarantees and so its result should be considered 'out of date' + /// the instant it is called. Do not use it for synchronization purposes. However, it may be useful as a heuristic. + pub fn reader_count(&self) -> usize { + let state = self.lock.load(Ordering::Relaxed); + state / READER + (state & UPGRADED) / UPGRADED + } + + /// Return the number of writers that currently hold the lock. + /// + /// Because [`RwLock`] guarantees exclusive mutable access, this function may only return either `0` or `1`. + /// + /// # Safety + /// + /// This function provides no synchronization guarantees and so its result should be considered 'out of date' + /// the instant it is called. Do not use it for synchronization purposes. However, it may be useful as a heuristic. + pub fn writer_count(&self) -> usize { + (self.lock.load(Ordering::Relaxed) & WRITER) / WRITER + } + + /// Force decrement the reader count. + /// + /// # Safety + /// + /// This is *extremely* unsafe if there are outstanding `RwLockReadGuard`s + /// live, or if called more times than `read` has been called, but can be + /// useful in FFI contexts where the caller doesn't know how to deal with + /// RAII. The underlying atomic operation uses `Ordering::Release`. + #[inline] + pub unsafe fn force_read_decrement(&self) { + debug_assert!(self.lock.load(Ordering::Relaxed) & !WRITER > 0); + self.lock.fetch_sub(READER, Ordering::Release); + } + + /// Force unlock exclusive write access. + /// + /// # Safety + /// + /// This is *extremely* unsafe if there are outstanding `RwLockWriteGuard`s + /// live, or if called when there are current readers, but can be useful in + /// FFI contexts where the caller doesn't know how to deal with RAII. The + /// underlying atomic operation uses `Ordering::Release`. + #[inline] + pub unsafe fn force_write_unlock(&self) { + debug_assert_eq!(self.lock.load(Ordering::Relaxed) & !(WRITER | UPGRADED), 0); + self.lock.fetch_and(!(WRITER | UPGRADED), Ordering::Release); + } + + #[inline(always)] + fn try_write_internal(&self, strong: bool) -> Option> { + if compare_exchange( + &self.lock, + 0, + WRITER, + Ordering::Acquire, + Ordering::Relaxed, + strong, + ) + .is_ok() + { + Some(RwLockWriteGuard { + phantom: PhantomData, + inner: self, + data: unsafe { &mut *self.data.get() }, + }) + } else { + None + } + } + + /// Attempt to lock this rwlock with exclusive write access. + /// + /// This function does not ever block, and it will return `None` if a call + /// to `write` would otherwise block. If successful, an RAII guard is + /// returned. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// { + /// match mylock.try_write() { + /// Some(mut data) => { + /// // The lock is now locked and the data can be written + /// *data += 1; + /// // The lock is implicitly dropped + /// } + /// None => (), // no cigar + /// }; + /// } + /// ``` + #[inline] + pub fn try_write(&self) -> Option> { + self.try_write_internal(true) + } + + /// Attempt to lock this rwlock with exclusive write access. + /// + /// Unlike [`RwLock::try_write`], this function is allowed to spuriously fail even when acquiring exclusive write access + /// would otherwise succeed, which can result in more efficient code on some platforms. + #[inline] + pub fn try_write_weak(&self) -> Option> { + self.try_write_internal(false) + } + + /// Tries to obtain an upgradeable lock guard. + #[inline] + pub fn try_upgradeable_read(&self) -> Option> { + if self.lock.fetch_or(UPGRADED, Ordering::Acquire) & (WRITER | UPGRADED) == 0 { + Some(RwLockUpgradableGuard { + phantom: PhantomData, + inner: self, + data: unsafe { &*self.data.get() }, + }) + } else { + // We can't unflip the UPGRADED bit back just yet as there is another upgradeable or write lock. + // When they unlock, they will clear the bit. + None + } + } + + /// Returns a mutable reference to the underlying data. + /// + /// Since this call borrows the `RwLock` mutably, no actual locking needs to + /// take place -- the mutable borrow statically guarantees no locks exist. + /// + /// # Examples + /// + /// ``` + /// let mut lock = spin::RwLock::new(0); + /// *lock.get_mut() = 10; + /// assert_eq!(*lock.read(), 10); + /// ``` + pub fn get_mut(&mut self) -> &mut T { + // We know statically that there are no other references to `self`, so + // there's no need to lock the inner lock. + unsafe { &mut *self.data.get() } + } +} + +impl fmt::Debug for RwLock { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.try_read() { + Some(guard) => write!(f, "RwLock {{ data: ") + .and_then(|()| (*guard).fmt(f)) + .and_then(|()| write!(f, " }}")), + None => write!(f, "RwLock {{ }}"), + } + } +} + +impl Default for RwLock { + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl From for RwLock { + fn from(data: T) -> Self { + Self::new(data) + } +} + +impl<'rwlock, T: ?Sized> RwLockReadGuard<'rwlock, T> { + /// Leak the lock guard, yielding a reference to the underlying data. + /// + /// Note that this function will permanently lock the original lock for all but reading locks. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// + /// let data: &i32 = spin::RwLockReadGuard::leak(mylock.read()); + /// + /// assert_eq!(*data, 0); + /// ``` + #[inline] + pub fn leak(this: Self) -> &'rwlock T { + let this = ManuallyDrop::new(this); + // Safety: We know statically that only we are referencing data + unsafe { &*this.data } + } +} + +impl<'rwlock, T: ?Sized + fmt::Debug> fmt::Debug for RwLockReadGuard<'rwlock, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<'rwlock, T: ?Sized + fmt::Display> fmt::Display for RwLockReadGuard<'rwlock, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<'rwlock, T: ?Sized, R: RelaxStrategy> RwLockUpgradableGuard<'rwlock, T, R> { + /// Upgrades an upgradeable lock guard to a writable lock guard. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// + /// let upgradeable = mylock.upgradeable_read(); // Readable, but not yet writable + /// let writable = upgradeable.upgrade(); + /// ``` + #[inline] + pub fn upgrade(mut self) -> RwLockWriteGuard<'rwlock, T, R> { + loop { + self = match self.try_upgrade_internal(false) { + Ok(guard) => return guard, + Err(e) => e, + }; + + R::relax(); + } + } +} + +impl<'rwlock, T: ?Sized, R> RwLockUpgradableGuard<'rwlock, T, R> { + #[inline(always)] + fn try_upgrade_internal(self, strong: bool) -> Result, Self> { + if compare_exchange( + &self.inner.lock, + UPGRADED, + WRITER, + Ordering::Acquire, + Ordering::Relaxed, + strong, + ) + .is_ok() + { + let inner = self.inner; + + // Forget the old guard so its destructor doesn't run (before mutably aliasing data below) + mem::forget(self); + + // Upgrade successful + Ok(RwLockWriteGuard { + phantom: PhantomData, + inner, + data: unsafe { &mut *inner.data.get() }, + }) + } else { + Err(self) + } + } + + /// Tries to upgrade an upgradeable lock guard to a writable lock guard. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// let upgradeable = mylock.upgradeable_read(); // Readable, but not yet writable + /// + /// match upgradeable.try_upgrade() { + /// Ok(writable) => + /// // upgrade successful - use writable lock guard + /// { + /// () + /// } + /// Err(upgradeable) => + /// // upgrade unsuccessful + /// { + /// () + /// } + /// }; + /// ``` + #[inline] + pub fn try_upgrade(self) -> Result, Self> { + self.try_upgrade_internal(true) + } + + /// Tries to upgrade an upgradeable lock guard to a writable lock guard. + /// + /// Unlike [`RwLockUpgradableGuard::try_upgrade`], this function is allowed to spuriously fail even when upgrading + /// would otherwise succeed, which can result in more efficient code on some platforms. + #[inline] + pub fn try_upgrade_weak(self) -> Result, Self> { + self.try_upgrade_internal(false) + } + + #[inline] + /// Downgrades the upgradeable lock guard to a readable, shared lock guard. Cannot fail and is guaranteed not to spin. + /// + /// ``` + /// let mylock = spin::RwLock::new(1); + /// + /// let upgradeable = mylock.upgradeable_read(); + /// assert!(mylock.try_read().is_none()); + /// assert_eq!(*upgradeable, 1); + /// + /// let readable = upgradeable.downgrade(); // This is guaranteed not to spin + /// assert!(mylock.try_read().is_some()); + /// assert_eq!(*readable, 1); + /// ``` + pub fn downgrade(self) -> RwLockReadGuard<'rwlock, T> { + // Reserve the read guard for ourselves + self.inner.acquire_reader(); + + let inner = self.inner; + + // Dropping self removes the UPGRADED bit + mem::drop(self); + + RwLockReadGuard { + lock: &inner.lock, + data: unsafe { &*inner.data.get() }, + } + } + + /// Leak the lock guard, yielding a reference to the underlying data. + /// + /// Note that this function will permanently lock the original lock. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// + /// let data: &i32 = spin::RwLockUpgradableGuard::leak(mylock.upgradeable_read()); + /// + /// assert_eq!(*data, 0); + /// ``` + #[inline] + pub fn leak(this: Self) -> &'rwlock T { + let this = ManuallyDrop::new(this); + // Safety: We know statically that only we are referencing data + unsafe { &*this.data } + } +} + +impl<'rwlock, T: ?Sized + fmt::Debug, R> fmt::Debug for RwLockUpgradableGuard<'rwlock, T, R> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<'rwlock, T: ?Sized + fmt::Display, R> fmt::Display for RwLockUpgradableGuard<'rwlock, T, R> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<'rwlock, T: ?Sized, R> RwLockWriteGuard<'rwlock, T, R> { + /// Downgrades the writable lock guard to a readable, shared lock guard. Cannot fail and is guaranteed not to spin. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// + /// let mut writable = mylock.write(); + /// *writable = 1; + /// + /// let readable = writable.downgrade(); // This is guaranteed not to spin + /// // + /// # let readable_2 = mylock.try_read().unwrap(); + /// assert_eq!(*readable, 1); + /// ``` + #[inline] + pub fn downgrade(self) -> RwLockReadGuard<'rwlock, T> { + // Reserve the read guard for ourselves + self.inner.acquire_reader(); + + let inner = self.inner; + + // Dropping self removes the UPGRADED bit + mem::drop(self); + + RwLockReadGuard { + lock: &inner.lock, + data: unsafe { &*inner.data.get() }, + } + } + + /// Downgrades the writable lock guard to an upgradable, shared lock guard. Cannot fail and is guaranteed not to spin. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// + /// let mut writable = mylock.write(); + /// *writable = 1; + /// + /// let readable = writable.downgrade_to_upgradeable(); // This is guaranteed not to spin + /// assert_eq!(*readable, 1); + /// ``` + #[inline] + pub fn downgrade_to_upgradeable(self) -> RwLockUpgradableGuard<'rwlock, T, R> { + debug_assert_eq!( + self.inner.lock.load(Ordering::Acquire) & (WRITER | UPGRADED), + WRITER + ); + + // Reserve the read guard for ourselves + self.inner.lock.store(UPGRADED, Ordering::Release); + + let inner = self.inner; + + // Dropping self removes the UPGRADED bit + mem::forget(self); + + RwLockUpgradableGuard { + phantom: PhantomData, + inner, + data: unsafe { &*inner.data.get() }, + } + } + + /// Leak the lock guard, yielding a mutable reference to the underlying data. + /// + /// Note that this function will permanently lock the original lock. + /// + /// ``` + /// let mylock = spin::RwLock::new(0); + /// + /// let data: &mut i32 = spin::RwLockWriteGuard::leak(mylock.write()); + /// + /// *data = 1; + /// assert_eq!(*data, 1); + /// ``` + #[inline] + pub fn leak(this: Self) -> &'rwlock mut T { + let mut this = ManuallyDrop::new(this); + // Safety: We know statically that only we are referencing data + unsafe { &mut *this.data } + } +} + +impl<'rwlock, T: ?Sized + fmt::Debug, R> fmt::Debug for RwLockWriteGuard<'rwlock, T, R> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<'rwlock, T: ?Sized + fmt::Display, R> fmt::Display for RwLockWriteGuard<'rwlock, T, R> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<'rwlock, T: ?Sized> Deref for RwLockReadGuard<'rwlock, T> { + type Target = T; + + fn deref(&self) -> &T { + // Safety: We know statically that only we are referencing data + unsafe { &*self.data } + } +} + +impl<'rwlock, T: ?Sized, R> Deref for RwLockUpgradableGuard<'rwlock, T, R> { + type Target = T; + + fn deref(&self) -> &T { + // Safety: We know statically that only we are referencing data + unsafe { &*self.data } + } +} + +impl<'rwlock, T: ?Sized, R> Deref for RwLockWriteGuard<'rwlock, T, R> { + type Target = T; + + fn deref(&self) -> &T { + // Safety: We know statically that only we are referencing data + unsafe { &*self.data } + } +} + +impl<'rwlock, T: ?Sized, R> DerefMut for RwLockWriteGuard<'rwlock, T, R> { + fn deref_mut(&mut self) -> &mut T { + // Safety: We know statically that only we are referencing data + unsafe { &mut *self.data } + } +} + +impl<'rwlock, T: ?Sized> Drop for RwLockReadGuard<'rwlock, T> { + fn drop(&mut self) { + debug_assert!(self.lock.load(Ordering::Relaxed) & !(WRITER | UPGRADED) > 0); + self.lock.fetch_sub(READER, Ordering::Release); + } +} + +impl<'rwlock, T: ?Sized, R> Drop for RwLockUpgradableGuard<'rwlock, T, R> { + fn drop(&mut self) { + debug_assert_eq!( + self.inner.lock.load(Ordering::Relaxed) & (WRITER | UPGRADED), + UPGRADED + ); + self.inner.lock.fetch_sub(UPGRADED, Ordering::AcqRel); + } +} + +impl<'rwlock, T: ?Sized, R> Drop for RwLockWriteGuard<'rwlock, T, R> { + fn drop(&mut self) { + debug_assert_eq!(self.inner.lock.load(Ordering::Relaxed) & WRITER, WRITER); + + // Writer is responsible for clearing both WRITER and UPGRADED bits. + // The UPGRADED bit may be set if an upgradeable lock attempts an upgrade while this lock is held. + self.inner + .lock + .fetch_and(!(WRITER | UPGRADED), Ordering::Release); + } +} + +#[inline(always)] +fn compare_exchange( + atomic: &AtomicUsize, + current: usize, + new: usize, + success: Ordering, + failure: Ordering, + strong: bool, +) -> Result { + if strong { + atomic.compare_exchange(current, new, success, failure) + } else { + atomic.compare_exchange_weak(current, new, success, failure) + } +} + +#[cfg(feature = "lock_api")] +unsafe impl lock_api_crate::RawRwLock for RwLock<(), R> { + type GuardMarker = lock_api_crate::GuardSend; + + const INIT: Self = Self::new(()); + + #[inline(always)] + fn lock_exclusive(&self) { + // Prevent guard destructor running + core::mem::forget(self.write()); + } + + #[inline(always)] + fn try_lock_exclusive(&self) -> bool { + // Prevent guard destructor running + self.try_write().map(core::mem::forget).is_some() + } + + #[inline(always)] + unsafe fn unlock_exclusive(&self) { + drop(RwLockWriteGuard { + inner: self, + data: &mut (), + phantom: PhantomData, + }); + } + + #[inline(always)] + fn lock_shared(&self) { + // Prevent guard destructor running + core::mem::forget(self.read()); + } + + #[inline(always)] + fn try_lock_shared(&self) -> bool { + // Prevent guard destructor running + self.try_read().map(core::mem::forget).is_some() + } + + #[inline(always)] + unsafe fn unlock_shared(&self) { + drop(RwLockReadGuard { + lock: &self.lock, + data: &(), + }); + } + + #[inline(always)] + fn is_locked(&self) -> bool { + self.lock.load(Ordering::Relaxed) != 0 + } +} + +#[cfg(feature = "lock_api")] +unsafe impl lock_api_crate::RawRwLockUpgrade for RwLock<(), R> { + #[inline(always)] + fn lock_upgradable(&self) { + // Prevent guard destructor running + core::mem::forget(self.upgradeable_read()); + } + + #[inline(always)] + fn try_lock_upgradable(&self) -> bool { + // Prevent guard destructor running + self.try_upgradeable_read().map(core::mem::forget).is_some() + } + + #[inline(always)] + unsafe fn unlock_upgradable(&self) { + drop(RwLockUpgradableGuard { + inner: self, + data: &(), + phantom: PhantomData, + }); + } + + #[inline(always)] + unsafe fn upgrade(&self) { + let tmp_guard = RwLockUpgradableGuard { + inner: self, + data: &(), + phantom: PhantomData, + }; + core::mem::forget(tmp_guard.upgrade()); + } + + #[inline(always)] + unsafe fn try_upgrade(&self) -> bool { + let tmp_guard = RwLockUpgradableGuard { + inner: self, + data: &(), + phantom: PhantomData, + }; + tmp_guard.try_upgrade().map(core::mem::forget).is_ok() + } +} + +#[cfg(feature = "lock_api")] +unsafe impl lock_api_crate::RawRwLockDowngrade for RwLock<(), R> { + unsafe fn downgrade(&self) { + let tmp_guard = RwLockWriteGuard { + inner: self, + data: &mut (), + phantom: PhantomData, + }; + core::mem::forget(tmp_guard.downgrade()); + } +} + +#[cfg(feature = "lock_api")] +unsafe impl lock_api_crate::RawRwLockUpgradeDowngrade for RwLock<(), R> { + unsafe fn downgrade_upgradable(&self) { + let tmp_guard = RwLockUpgradableGuard { + inner: self, + data: &(), + phantom: PhantomData, + }; + core::mem::forget(tmp_guard.downgrade()); + } + + unsafe fn downgrade_to_upgradable(&self) { + let tmp_guard = RwLockWriteGuard { + inner: self, + data: &mut (), + phantom: PhantomData, + }; + core::mem::forget(tmp_guard.downgrade_to_upgradeable()); + } +} + +#[cfg(test)] +mod tests { + use std::{ + prelude::v1::*, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + mpsc::channel, + }, + thread, + }; + + type RwLock = super::RwLock; + + #[derive(Eq, PartialEq, Debug)] + struct NonCopy(i32); + + #[test] + fn smoke() { + let l = RwLock::new(()); + drop(l.read()); + drop(l.write()); + drop((l.read(), l.read())); + drop(l.write()); + } + + // TODO: needs RNG + //#[test] + // fn frob() { + // static R: RwLock = RwLock::new(); + // const N: usize = 10; + // const M: usize = 1000; + // + // let (tx, rx) = channel::<()>(); + // for _ in 0..N { + // let tx = tx.clone(); + // thread::spawn(move|| { + // let mut rng = rand::thread_rng(); + // for _ in 0..M { + // if rng.gen_weighted_bool(N) { + // drop(R.write()); + // } else { + // drop(R.read()); + // } + // } + // drop(tx); + // }); + // } + // drop(tx); + // let _ = rx.recv(); + // unsafe { R.destroy(); } + //} + + #[test] + fn test_rw_arc() { + let arc = Arc::new(RwLock::new(0)); + let arc2 = arc.clone(); + let (tx, rx) = channel(); + + let t = thread::spawn(move || { + let mut lock = arc2.write(); + for _ in 0..10 { + let tmp = *lock; + *lock = -1; + thread::yield_now(); + *lock = tmp + 1; + } + tx.send(()).unwrap(); + }); + + // Readers try to catch the writer in the act + let mut children = Vec::new(); + for _ in 0..5 { + let arc3 = arc.clone(); + children.push(thread::spawn(move || { + let lock = arc3.read(); + assert!(*lock >= 0); + })); + } + + // Wait for children to pass their asserts + for r in children { + assert!(r.join().is_ok()); + } + + // Wait for writer to finish + rx.recv().unwrap(); + let lock = arc.read(); + assert_eq!(*lock, 10); + + assert!(t.join().is_ok()); + } + + #[test] + fn test_rw_access_in_unwind() { + let arc = Arc::new(RwLock::new(1)); + let arc2 = arc.clone(); + let _ = thread::spawn(move || -> () { + struct Unwinder { + i: Arc>, + } + impl Drop for Unwinder { + fn drop(&mut self) { + let mut lock = self.i.write(); + *lock += 1; + } + } + let _u = Unwinder { i: arc2 }; + panic!(); + }) + .join(); + let lock = arc.read(); + assert_eq!(*lock, 2); + } + + #[test] + fn test_rwlock_unsized() { + let rw: &RwLock<[i32]> = &RwLock::new([1, 2, 3]); + { + let b = &mut *rw.write(); + b[0] = 4; + b[2] = 5; + } + let comp: &[i32] = &[4, 2, 5]; + assert_eq!(&*rw.read(), comp); + } + + #[test] + fn test_rwlock_try_write() { + use std::mem::drop; + + let lock = RwLock::new(0isize); + let read_guard = lock.read(); + + let write_result = lock.try_write(); + match write_result { + None => (), + Some(_) => assert!( + false, + "try_write should not succeed while read_guard is in scope" + ), + } + + drop(read_guard); + } + + #[test] + fn test_rw_try_read() { + let m = RwLock::new(0); + ::std::mem::forget(m.write()); + assert!(m.try_read().is_none()); + } + + #[test] + fn test_into_inner() { + let m = RwLock::new(NonCopy(10)); + assert_eq!(m.into_inner(), NonCopy(10)); + } + + #[test] + fn test_into_inner_drop() { + struct Foo(Arc); + impl Drop for Foo { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let num_drops = Arc::new(AtomicUsize::new(0)); + let m = RwLock::new(Foo(num_drops.clone())); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + { + let _inner = m.into_inner(); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + } + assert_eq!(num_drops.load(Ordering::SeqCst), 1); + } + + #[test] + fn test_force_read_decrement() { + let m = RwLock::new(()); + ::std::mem::forget(m.read()); + ::std::mem::forget(m.read()); + ::std::mem::forget(m.read()); + assert!(m.try_write().is_none()); + unsafe { + m.force_read_decrement(); + m.force_read_decrement(); + } + assert!(m.try_write().is_none()); + unsafe { + m.force_read_decrement(); + } + assert!(m.try_write().is_some()); + } + + #[test] + fn test_force_write_unlock() { + let m = RwLock::new(()); + ::std::mem::forget(m.write()); + assert!(m.try_read().is_none()); + unsafe { + m.force_write_unlock(); + } + assert!(m.try_read().is_some()); + } + + #[test] + fn test_upgrade_downgrade() { + let m = RwLock::new(()); + { + let _r = m.read(); + let upg = m.try_upgradeable_read().unwrap(); + assert!(m.try_read().is_none()); + assert!(m.try_write().is_none()); + assert!(upg.try_upgrade().is_err()); + } + { + let w = m.write(); + assert!(m.try_upgradeable_read().is_none()); + let _r = w.downgrade(); + assert!(m.try_upgradeable_read().is_some()); + assert!(m.try_read().is_some()); + assert!(m.try_write().is_none()); + } + { + let _u = m.upgradeable_read(); + assert!(m.try_upgradeable_read().is_none()); + } + + assert!(m.try_upgradeable_read().unwrap().try_upgrade().is_ok()); + } +} diff --git a/components/x86_vcpu/Cargo.toml b/components/x86_vcpu/Cargo.toml index b90a91cd8f..fee64730b9 100644 --- a/components/x86_vcpu/Cargo.toml +++ b/components/x86_vcpu/Cargo.toml @@ -36,9 +36,9 @@ axvcpu = { workspace = true } x86_vlapic = { workspace = true } axdevice_base = { workspace = true } axvisor_api = { workspace = true } -spin = { version = "0.10", default-features = false } [dev-dependencies] +ax-kspin = { workspace = true } memoffset = "0.9" [features] diff --git a/components/x86_vcpu/src/test_utils.rs b/components/x86_vcpu/src/test_utils.rs index cebcd7b767..7d091326d5 100644 --- a/components/x86_vcpu/src/test_utils.rs +++ b/components/x86_vcpu/src/test_utils.rs @@ -14,9 +14,9 @@ #[cfg(test)] pub mod mock { + use ax_kspin::SpinNoIrq as Mutex; use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr, VirtAddr}; use axvisor_api::{api_impl, memory::MemoryIf}; - use spin::Mutex; static GLOBAL_LOCK: Mutex = Mutex::new(MockMmHalState::new()); static TEST_LOCK: Mutex<()> = Mutex::new(()); diff --git a/drivers/blk/nvme-driver/Cargo.toml b/drivers/blk/nvme-driver/Cargo.toml index 9dfdb91642..33f56b33e8 100644 --- a/drivers/blk/nvme-driver/Cargo.toml +++ b/drivers/blk/nvme-driver/Cargo.toml @@ -11,11 +11,11 @@ repository.workspace = true version = "0.4.2" [dependencies] +ax-kspin.workspace = true dma-api.workspace = true log = "0.4" mmio-api.workspace = true rd-block.workspace = true -spin = "0.10" tock-registers = "0.10" [dev-dependencies] diff --git a/drivers/blk/nvme-driver/src/block.rs b/drivers/blk/nvme-driver/src/block.rs index 5288bbf404..9be6af7124 100644 --- a/drivers/blk/nvme-driver/src/block.rs +++ b/drivers/blk/nvme-driver/src/block.rs @@ -1,11 +1,11 @@ use alloc::{boxed::Box, collections::BTreeSet, sync::Arc}; use core::any::Any; +use ax_kspin::SpinNoIrq as Mutex; use rd_block::{ BlkError, Block as RdBlock, BuffConfig, DriverGeneric, Event, IQueue, IdList, Interface, Request, RequestId, RequestKind, }; -use spin::Mutex; use crate::{Namespace, Nvme, err::Result}; diff --git a/drivers/blk/ramdisk/Cargo.toml b/drivers/blk/ramdisk/Cargo.toml index 8ea60dfcd6..322cc13e41 100644 --- a/drivers/blk/ramdisk/Cargo.toml +++ b/drivers/blk/ramdisk/Cargo.toml @@ -11,8 +11,8 @@ version = "0.1.1" publish = false [dependencies] +ax-kspin = { workspace = true } rdif-block = { workspace = true } -spin = "0.10" [dev-dependencies] dma-api = { workspace = true } diff --git a/drivers/blk/ramdisk/src/lib.rs b/drivers/blk/ramdisk/src/lib.rs index b27148d4bc..510c275b70 100644 --- a/drivers/blk/ramdisk/src/lib.rs +++ b/drivers/blk/ramdisk/src/lib.rs @@ -4,11 +4,11 @@ extern crate alloc; use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use ax_kspin::SpinNoIrq as Mutex; use rdif_block::{ BlkError, BuffConfig, DriverGeneric, Event, IQueue, IdList, Interface, Request, RequestId, RequestKind, }; -use spin::Mutex; struct RamInner { storage: Vec, diff --git a/drivers/firmware/arm-scmi-rs/Cargo.toml b/drivers/firmware/arm-scmi-rs/Cargo.toml index ef420e31c4..fa43b92472 100644 --- a/drivers/firmware/arm-scmi-rs/Cargo.toml +++ b/drivers/firmware/arm-scmi-rs/Cargo.toml @@ -11,13 +11,13 @@ keywords = ["arm", "scmi", "no_std", "embedded", "clock"] categories = ["embedded", "hardware-support", "no-std", "os"] [dependencies] +ax-kspin = { workspace = true } bitflags = "2" log = "0.4" mbarrier = "0.1" smccc = "0.2" thiserror = {version = "2", default-features = false} tock-registers = "0.10" -spin = "0.10" nb = "1" aarch64-cpu-ext = "0.1" diff --git a/drivers/firmware/arm-scmi-rs/src/lib.rs b/drivers/firmware/arm-scmi-rs/src/lib.rs index 055db17146..6101ce38d6 100644 --- a/drivers/firmware/arm-scmi-rs/src/lib.rs +++ b/drivers/firmware/arm-scmi-rs/src/lib.rs @@ -48,7 +48,7 @@ mod transport; use alloc::sync::Arc; -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; pub use transport::{Smc, Transport}; type Data = Arc>>; diff --git a/drivers/firmware/arm-scmi-rs/src/protocol/mod.rs b/drivers/firmware/arm-scmi-rs/src/protocol/mod.rs index 767e569a74..50eb34015d 100644 --- a/drivers/firmware/arm-scmi-rs/src/protocol/mod.rs +++ b/drivers/firmware/arm-scmi-rs/src/protocol/mod.rs @@ -1,8 +1,8 @@ use alloc::vec::Vec; use core::sync::atomic::{AtomicI32, Ordering}; +use ax_kspin::SpinNoIrq as Mutex; use mbarrier::smp_mb; -use spin::Mutex; use crate::{Data, Transport, err::ScmiError}; diff --git a/drivers/interface/rdif-serial/Cargo.toml b/drivers/interface/rdif-serial/Cargo.toml index 6fd6f53d90..ce593b0da7 100644 --- a/drivers/interface/rdif-serial/Cargo.toml +++ b/drivers/interface/rdif-serial/Cargo.toml @@ -10,9 +10,9 @@ repository.workspace = true version = "0.7.1" [dependencies] +ax-kspin.workspace = true bitflags = "2.8" futures = {version = "0.3", default-features = false, features = ["alloc"]} rdif-base = {workspace = true} -spin = "0.10" thiserror = {version = "2", default-features = false} heapless = "0.9" diff --git a/drivers/interface/rdif-serial/src/serial.rs b/drivers/interface/rdif-serial/src/serial.rs index 8d6d7d106a..c329b3e1a4 100644 --- a/drivers/interface/rdif-serial/src/serial.rs +++ b/drivers/interface/rdif-serial/src/serial.rs @@ -1,9 +1,9 @@ use alloc::{boxed::Box, sync::Arc}; use core::{cell::UnsafeCell, num::NonZeroU32}; +use ax_kspin::SpinNoIrq as Mutex; use heapless::Deque; use rdif_base::DriverGeneric; -use spin::Mutex; use super::{ BIrqHandler, BReciever, BSender, BSerial, InterfaceRaw, InterruptMask, TransBytesError, diff --git a/drivers/net/realtek-rtl8125/Cargo.toml b/drivers/net/realtek-rtl8125/Cargo.toml index 72156b29ec..af62780e2a 100644 --- a/drivers/net/realtek-rtl8125/Cargo.toml +++ b/drivers/net/realtek-rtl8125/Cargo.toml @@ -10,10 +10,10 @@ license = "MIT" repository.workspace = true [dependencies] +ax-kspin.workspace = true dma-api.workspace = true log.workspace = true mmio-api.workspace = true rdif-eth.workspace = true -spin.workspace = true thiserror = { version = "2", default-features = false } tock-registers = "0.10" diff --git a/drivers/net/realtek-rtl8125/src/lib.rs b/drivers/net/realtek-rtl8125/src/lib.rs index 805cd0200b..f120501118 100644 --- a/drivers/net/realtek-rtl8125/src/lib.rs +++ b/drivers/net/realtek-rtl8125/src/lib.rs @@ -5,13 +5,13 @@ extern crate alloc; use alloc::{boxed::Box, collections::VecDeque, sync::Arc}; use core::sync::atomic::{Ordering as AtomicOrdering, fence}; +use ax_kspin::SpinNoIrq as Mutex; use descriptor::{RING_END, RxDesc, TxDesc}; use dma_api::{DArray, DeviceDma, DmaDirection, DmaOp}; use log::{debug, info, warn}; use mmio_api::{Mmio, MmioAddr, MmioOp}; use rdif_eth::{Event, IRxQueue, ITxQueue, Interface, NetError, QueueConfig}; use registers::*; -use spin::Mutex; mod descriptor; mod registers; diff --git a/drivers/rdrive/Cargo.toml b/drivers/rdrive/Cargo.toml index 08aa0560cc..5f6005ba54 100644 --- a/drivers/rdrive/Cargo.toml +++ b/drivers/rdrive/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true version = "0.20.1" [dependencies] +ax-kspin.workspace = true fdt-edit.workspace = true fdt-raw.workspace = true log = "0.4" diff --git a/drivers/rdrive/src/lib.rs b/drivers/rdrive/src/lib.rs index 60b0b5e6f6..0cb1698408 100644 --- a/drivers/rdrive/src/lib.rs +++ b/drivers/rdrive/src/lib.rs @@ -7,9 +7,10 @@ extern crate log; use core::ptr::NonNull; +use ax_kspin::SpinNoIrq as Mutex; pub use fdt_edit::Phandle; use register::{DriverRegister, ProbeLevel}; -use spin::{Mutex, Once}; +use spin::Once; mod descriptor; pub mod driver; diff --git a/drivers/rdrive/src/probe/fdt/mod.rs b/drivers/rdrive/src/probe/fdt/mod.rs index 07ec1b595e..17c1269a5e 100644 --- a/drivers/rdrive/src/probe/fdt/mod.rs +++ b/drivers/rdrive/src/probe/fdt/mod.rs @@ -4,8 +4,9 @@ use alloc::{ }; use core::ptr::NonNull; +use ax_kspin::SpinNoIrq as Mutex; pub use fdt_edit::{ClockRef, Fdt, InterruptRef, NodeType, Phandle, RegInfo, Status}; -use spin::{Mutex, Once}; +use spin::Once; use super::ProbeError; use crate::{ diff --git a/drivers/rdrive/src/probe/pci/mod.rs b/drivers/rdrive/src/probe/pci/mod.rs index 0369478478..09db562266 100644 --- a/drivers/rdrive/src/probe/pci/mod.rs +++ b/drivers/rdrive/src/probe/pci/mod.rs @@ -3,9 +3,10 @@ use core::ops::{Deref, DerefMut}; use ::pcie::*; pub use ::pcie::{Endpoint, PciCapability, PcieGeneric}; +use ax_kspin::SpinNoIrq as Mutex; use mmio_api::{MapError, MmioOp}; pub use rdif_pcie::{DriverGeneric, PciAddress, PciMem32, PciMem64, PcieController}; -use spin::{Mutex, Once}; +use spin::Once; use crate::{ Descriptor, Device, PlatformDevice, ProbeError, get_list, diff --git a/drivers/serial/some-serial/Cargo.toml b/drivers/serial/some-serial/Cargo.toml index 21269117ab..e8de4bd90c 100644 --- a/drivers/serial/some-serial/Cargo.toml +++ b/drivers/serial/some-serial/Cargo.toml @@ -24,10 +24,10 @@ tock-registers.workspace = true x86.workspace = true [target.'cfg(target_os = "none")'.dev-dependencies] +ax-kspin.workspace = true fdt-edit.workspace = true rdif-intc.workspace = true rdrive.workspace = true -spin.workspace = true # bare-test integration test is temporarily disabled. # [[test]] diff --git a/drivers/serial/some-serial/tests/test.rs b/drivers/serial/some-serial/tests/test.rs index 47be8b9dce..85a7b567f3 100644 --- a/drivers/serial/some-serial/tests/test.rs +++ b/drivers/serial/some-serial/tests/test.rs @@ -22,11 +22,11 @@ mod tests { }, }; use fdt_edit::{ClockType, Fdt, InterruptRef, NodeType}; + use ax_kspin::SpinNoIrq as Mutex; use rdif_intc::Intc; use rdif_serial::{BIrqHandler, BReciever, BSender, BSerial, TransferError}; use rdrive::fdt_phandle_to_device_id; use some_serial::{Config, DataBits, InterruptMask, Parity, StopBits}; - use spin::Mutex; static TX_INTERRUPT_COUNT: AtomicUsize = AtomicUsize::new(0); static RX_INTERRUPT_COUNT: AtomicUsize = AtomicUsize::new(0); diff --git a/drivers/soc/rockchip/rockchip-soc/Cargo.toml b/drivers/soc/rockchip/rockchip-soc/Cargo.toml index 91a508c6da..0609a40a95 100644 --- a/drivers/soc/rockchip/rockchip-soc/Cargo.toml +++ b/drivers/soc/rockchip/rockchip-soc/Cargo.toml @@ -22,6 +22,7 @@ thiserror = {version = "2", default-features = false} tock-registers = "0.10" [target.'cfg(not(any(windows, unix)))'.dev-dependencies] +ax-kspin.workspace = true num-align = "0.1.0" spin = "0.10" diff --git a/drivers/soc/rockchip/rockchip-soc/tests/test.rs b/drivers/soc/rockchip/rockchip-soc/tests/test.rs index 14034545d1..33a7cf6a3f 100644 --- a/drivers/soc/rockchip/rockchip-soc/tests/test.rs +++ b/drivers/soc/rockchip/rockchip-soc/tests/test.rs @@ -9,10 +9,11 @@ mod pin; #[bare_test::tests] mod tests { + use ax_kspin::SpinNoIrq as Mutex; use bare_test::mem::iomap; use log::info; use rockchip_soc::{Cru, SocType}; - use spin::{Mutex, Once}; + use spin::Once; use crate::pin::test_pin; diff --git a/drivers/tpu/sg2002-tpu/Cargo.toml b/drivers/tpu/sg2002-tpu/Cargo.toml index e95bb9f593..cdcb8f74b7 100644 --- a/drivers/tpu/sg2002-tpu/Cargo.toml +++ b/drivers/tpu/sg2002-tpu/Cargo.toml @@ -12,6 +12,6 @@ keywords = ["sg2002", "cvitek", "tpu", "ion"] [dependencies] ax-dma = { workspace = true } ax-errno = { workspace = true } +ax-kspin.workspace = true ax-memory-addr = { workspace = true } log = "0.4" -spin = "0.10" diff --git a/drivers/tpu/sg2002-tpu/src/ion/buffer.rs b/drivers/tpu/sg2002-tpu/src/ion/buffer.rs index a3ed44ff67..31debefb78 100644 --- a/drivers/tpu/sg2002-tpu/src/ion/buffer.rs +++ b/drivers/tpu/sg2002-tpu/src/ion/buffer.rs @@ -2,7 +2,7 @@ use alloc::{collections::BTreeMap, sync::Arc}; -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; use super::{ error::{IonError, IonResult}, diff --git a/drivers/tpu/sg2002-tpu/src/tpu/device.rs b/drivers/tpu/sg2002-tpu/src/tpu/device.rs index a47e333a8d..04fadc1520 100644 --- a/drivers/tpu/sg2002-tpu/src/tpu/device.rs +++ b/drivers/tpu/sg2002-tpu/src/tpu/device.rs @@ -9,7 +9,7 @@ use core::{ sync::atomic::{AtomicU32, Ordering}, }; -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; use super::{ TDMA_PHYS_BASE, TIU_PHYS_BASE, error::TpuError, platform::TpuRuntimeState, tdma::TdmaRegs, @@ -80,7 +80,7 @@ struct TpuDeviceInner { /// SG2002 TPU 设备（仅硬件层） pub struct Sg2002Tpu { - /// 内部状态 (使用 Mutex 保护) + /// 内部状态 (使用自旋锁保护) inner: Mutex, /// 序列号计数器 seq_counter: AtomicU32, diff --git a/drivers/usb/usb-host/Cargo.toml b/drivers/usb/usb-host/Cargo.toml index b899f2f5ab..e418257f91 100644 --- a/drivers/usb/usb-host/Cargo.toml +++ b/drivers/usb/usb-host/Cargo.toml @@ -15,6 +15,7 @@ default = ["aggressive_usb_reset"] libusb = ["libusb1-sys"] [dependencies] +ax-kspin.workspace = true bitflags = "2.8" crossbeam = {version = "0.8", features = ["alloc"], default-features = false} crossbeam-skiplist = {version = "0.1", features = [ diff --git a/drivers/usb/usb-host/src/backend/kmod/xhci/cmd.rs b/drivers/usb/usb-host/src/backend/kmod/xhci/cmd.rs index 27f0aca6bb..5eb8b2f207 100644 --- a/drivers/usb/usb-host/src/backend/kmod/xhci/cmd.rs +++ b/drivers/usb/usb-host/src/backend/kmod/xhci/cmd.rs @@ -1,7 +1,8 @@ use alloc::sync::Arc; +use ax_kspin::SpinNoIrq as Mutex; use mbarrier::wmb; -use spin::{Mutex, RwLock}; +use spin::RwLock; use usb_if::err::TransferError; use xhci::{ registers::doorbell, diff --git a/drivers/usb/usb-host/src/backend/kmod/xhci/device.rs b/drivers/usb/usb-host/src/backend/kmod/xhci/device.rs index 34ac735380..c795ed559d 100644 --- a/drivers/usb/usb-host/src/backend/kmod/xhci/device.rs +++ b/drivers/usb/usb-host/src/backend/kmod/xhci/device.rs @@ -1,8 +1,8 @@ use alloc::{collections::BTreeMap, sync::Arc, vec, vec::Vec}; +use ax_kspin::SpinNoIrq as Mutex; use futures::{FutureExt, future::BoxFuture}; use mbarrier::mb; -use spin::Mutex; use usb_if::{ descriptor::{ ConfigurationDescriptor, DescriptorType, DeviceDescriptor, DeviceDescriptorBase, diff --git a/drivers/usb/usb-host/src/backend/kmod/xhci/endpoint.rs b/drivers/usb/usb-host/src/backend/kmod/xhci/endpoint.rs index 5c11453e58..f1fa8a7001 100644 --- a/drivers/usb/usb-host/src/backend/kmod/xhci/endpoint.rs +++ b/drivers/usb/usb-host/src/backend/kmod/xhci/endpoint.rs @@ -1,8 +1,8 @@ use alloc::{collections::BTreeMap, sync::Arc, vec, vec::Vec}; +use ax_kspin::SpinNoIrq as Mutex; use dma_api::DmaDirection; use mbarrier::mb; -use spin::Mutex; use usb_if::{ descriptor::{self, EndpointDescriptor}, endpoint::{RequestId, TransferCompletion, TransferRequest}, diff --git a/drivers/usb/usb-host/src/backend/kmod/xhci/sync.rs b/drivers/usb/usb-host/src/backend/kmod/xhci/sync.rs index 8052e81347..745cc25279 100644 --- a/drivers/usb/usb-host/src/backend/kmod/xhci/sync.rs +++ b/drivers/usb/usb-host/src/backend/kmod/xhci/sync.rs @@ -1,7 +1,8 @@ use alloc::sync::Arc; use core::cell::UnsafeCell; -use spin::{Mutex, RwLock}; +use ax_kspin::{SpinNoIrq as Mutex, SpinNoIrqGuard as MutexGuard}; +use spin::RwLock; use super::reg::{DisableIrqGuard, XhciRegisters}; @@ -40,7 +41,7 @@ impl IrqLock { } pub(crate) struct IrqLockGuard<'a, T> { - _guard: spin::MutexGuard<'a, ()>, + _guard: MutexGuard<'a, ()>, data: &'a mut T, _disable_guard: DisableIrqGuard, } diff --git a/os/StarryOS/kernel/Cargo.toml b/os/StarryOS/kernel/Cargo.toml index 0d89538010..0da1df1913 100644 --- a/os/StarryOS/kernel/Cargo.toml +++ b/os/StarryOS/kernel/Cargo.toml @@ -93,7 +93,6 @@ inherit-methods-macro = "0.1.0" ax-kernel-guard = { workspace = true } kernel-elf-parser = "0.3.4" ax-kspin = { workspace = true } -lazy_static = { version = "1.5", features = ["spin_no_std"] } linux-raw-sys = { version = "0.12", default-features = false, features = [ "no_std", "general", diff --git a/os/StarryOS/kernel/src/file/epoll.rs b/os/StarryOS/kernel/src/file/epoll.rs index aadf3e5be4..2402e0ec52 100644 --- a/os/StarryOS/kernel/src/file/epoll.rs +++ b/os/StarryOS/kernel/src/file/epoll.rs @@ -19,7 +19,7 @@ use core::{ }; use ax_errno::{AxError, AxResult}; -use ax_kspin::SpinNoPreempt; +use ax_kspin::SpinNoIrq; use axpoll::{IoEvents, PollSet, Pollable}; use bitflags::bitflags; use hashbrown::HashMap; @@ -145,7 +145,7 @@ impl Eq for EntryKey {} struct EpollInterest { key: EntryKey, event: EpollEvent, - mode: SpinNoPreempt, + mode: SpinNoIrq, in_ready_queue: AtomicBool, } @@ -154,7 +154,7 @@ impl EpollInterest { Self { key, event, - mode: SpinNoPreempt::new(TriggerMode::from_flags(flags)), + mode: SpinNoIrq::new(TriggerMode::from_flags(flags)), in_ready_queue: AtomicBool::new(false), } } @@ -241,6 +241,10 @@ impl Wake for InterestWaker { }; if interest.try_mark_in_queue() { + // The queue lock must disable IRQs because wakers may be invoked + // from IRQ wake paths. `VecDeque::push_back` can still allocate + // when capacity is exhausted; if this path is proven to run in IRQ + // context, replace the queue with a bounded or deferred design. epoll .ready_queue .lock() @@ -255,16 +259,16 @@ impl Wake for InterestWaker { } struct EpollInner { - interests: SpinNoPreempt>>, - ready_queue: SpinNoPreempt>>, + interests: SpinNoIrq>>, + ready_queue: SpinNoIrq>>, poll_ready: PollSet, } impl Default for EpollInner { fn default() -> Self { Self { - interests: SpinNoPreempt::new(HashMap::new()), - ready_queue: SpinNoPreempt::new(VecDeque::new()), + interests: SpinNoIrq::new(HashMap::new()), + ready_queue: SpinNoIrq::new(VecDeque::new()), poll_ready: PollSet::new(), } } diff --git a/os/StarryOS/kernel/src/file/netlink.rs b/os/StarryOS/kernel/src/file/netlink.rs index dc3787bc67..0f9b3fd403 100644 --- a/os/StarryOS/kernel/src/file/netlink.rs +++ b/os/StarryOS/kernel/src/file/netlink.rs @@ -32,15 +32,15 @@ use core::{ }; use ax_errno::{AxError, AxResult}; +use ax_kspin::SpinNoIrq as Mutex; use ax_task::future::{block_on, poll_io}; use axpoll::{IoEvents, PollSet, Pollable}; -use lazy_static::lazy_static; use linux_raw_sys::{ general::{O_RDWR, S_IFSOCK}, net::AF_NETLINK, netlink::{NETLINK_GENERIC, NETLINK_KOBJECT_UEVENT, NETLINK_ROUTE, sockaddr_nl}, }; -use spin::Mutex; +use spin::Lazy; use super::packet::{ETH0_HWADDR, ETH0_IFINDEX}; use crate::{ @@ -245,12 +245,11 @@ pub struct NetlinkSocket { queue: Mutex>>, } -lazy_static! { - /// Global registry of bound netlink sockets, used by [`broadcast`] - /// to dispatch kernel-side messages. Holds weak refs so socket close - /// drops naturally; dead entries are pruned on each broadcast. - static ref NETLINK_SOCKETS: Mutex>> = Mutex::new(Vec::new()); -} +/// Global registry of bound netlink sockets, used by [`broadcast`] to dispatch +/// kernel-side messages. Holds weak refs so socket close drops naturally; dead +/// entries are pruned on each broadcast. +static NETLINK_SOCKETS: Lazy>>> = + Lazy::new(|| Mutex::new(Vec::new())); impl NetlinkSocket { pub fn new(protocol: u32) -> Arc { @@ -416,13 +415,15 @@ impl NetlinkSocket { /// Drain at most one queued message into `dst`. Returns `WouldBlock` /// when the queue is empty. fn read_one(&self, dst: &mut IoDst) -> AxResult { - let mut queue = self.queue.lock(); - let Some(msg) = queue.front() else { - return Err(AxError::WouldBlock); + let msg = { + let mut queue = self.queue.lock(); + let Some(msg) = queue.pop_front() else { + return Err(AxError::WouldBlock); + }; + msg }; // Cap at the message length; netlink datagrams are not coalesced. - let n = dst.write(msg)?; - queue.pop_front(); + let n = dst.write(&msg)?; Ok(n) } } diff --git a/os/StarryOS/kernel/src/file/packet.rs b/os/StarryOS/kernel/src/file/packet.rs index eb26f34deb..e276ea4916 100644 --- a/os/StarryOS/kernel/src/file/packet.rs +++ b/os/StarryOS/kernel/src/file/packet.rs @@ -160,8 +160,12 @@ impl PacketSocket { pub fn recv_packet(&self, dst: &mut IoDst) -> AxResult<(usize, SockAddrLl)> { block_on(poll_io(self, IoEvents::IN, self.nonblocking(), || { - let Some(frame) = self.state.lock().pending.take() else { - return Err(AxError::WouldBlock); + let frame = { + let mut state = self.state.lock(); + let Some(frame) = state.pending.take() else { + return Err(AxError::WouldBlock); + }; + frame }; let written = dst.write(&frame.data)?; Ok((written, frame.from)) diff --git a/os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs b/os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs index 6c82a1d4a2..dd2698a881 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs @@ -4,6 +4,7 @@ use core::{any::Any, time::Duration}; use ax_errno::{AxError, LinuxError}; use ax_hal::mem::phys_to_virt; +use ax_kspin::SpinNoIrq as Mutex; use ax_memory_addr::PhysAddr; use ax_task::sleep; use axfs_ng_vfs::{NodeFlags, VfsResult}; @@ -11,7 +12,6 @@ use sg200x_bsp::{ pinmux::{FMUX_SD1_D1, FMUX_SD1_D2, Pinmux}, soc::{FMUX_BASE, IOBLK_BASE, IOBLK_GRTC_BASE}, }; -use spin::Mutex; use starry_vm::{VmMutPtr, vm_write_slice}; use tock_registers::interfaces::Writeable; diff --git a/os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs b/os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs index 5200c8335b..09a0ba5b61 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs @@ -6,6 +6,7 @@ use ax_hal::{ mem::{phys_to_virt, virt_to_phys}, time::busy_wait, }; +use ax_kspin::SpinNoIrq as Mutex; use ax_memory_addr::{PhysAddr, VirtAddr}; use axfs_ng_vfs::{NodeFlags, VfsResult}; use sg200x_bsp::{ @@ -22,7 +23,6 @@ use sg200x_bsp::{ host::{self, UvcEnumerated, dwc2, dwc2::ep0 as dwc2_ep0}, }, }; -use spin::Mutex; use starry_vm::{VmMutPtr, vm_write_slice}; use tock_registers::interfaces::Writeable; diff --git a/os/StarryOS/kernel/src/pseudofs/dev/loop.rs b/os/StarryOS/kernel/src/pseudofs/dev/loop.rs index c3c668e92c..b6d7172f27 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/loop.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/loop.rs @@ -8,7 +8,7 @@ use core::{ use ax_errno::{AxError, AxResult, LinuxError}; use ax_fs::FileBackend; #[cfg(feature = "ext4")] -use ax_kspin::SpinNoPreempt; +use ax_kspin::SpinNoIrq; use ax_sync::Mutex; use axfs_ng_vfs::{DeviceId, NodeFlags, VfsResult}; use linux_raw_sys::{ @@ -81,13 +81,13 @@ fn writeback_buffer(file: &FileBackend, cd: &CacheData) -> bool { /// Owning an `Arc` keeps the buffer alive even if the /// `LoopDevice` replaces its cache slot (e.g. on re-mount). /// -/// The buffer is protected by a `SpinNoPreempt` lock. Both block-device -/// I/O (`read_block`/`write_block`, already under ext4's SpinNoPreempt) -/// and write-back paths (normal syscall context) acquire this lock, -/// so no concurrent access is possible regardless of mount state. +/// The buffer is protected by `SpinNoIrq` because ext4 may call +/// `read_block`/`write_block` under filesystem locks that already disable IRQs. +/// Write-back copies each chunk out under this short guard, then performs VFS +/// I/O after dropping it. #[cfg(feature = "ext4")] struct CacheData { - blocks: SpinNoPreempt>>, + blocks: SpinNoIrq>>, total_len: usize, dirty: AtomicBool, /// `true` while a `LoopBlockDevice` referencing this cache is alive. @@ -98,7 +98,7 @@ struct CacheData { impl CacheData { fn new(blocks: alloc::vec::Vec>, total_len: usize) -> Self { Self { - blocks: SpinNoPreempt::new(blocks), + blocks: SpinNoIrq::new(blocks), total_len, dirty: AtomicBool::new(false), mounted: AtomicBool::new(false), @@ -122,8 +122,8 @@ impl CacheData { /// - `BLKFLSBUF` ioctl: explicit flush /// /// All three run in normal syscall context where VFS I/O is safe. -/// The `flush()` callback is intentionally a no-op because ext4 invokes it -/// inside `SpinNoPreempt`. +/// The `flush()` callback is intentionally a no-op because filesystem block +/// I/O paths must not re-enter backing-file VFS writeback. #[cfg(feature = "ext4")] pub struct LoopBlockDevice { cache: Arc, @@ -227,10 +227,10 @@ impl ax_driver::prelude::BlockDriverOps for LoopBlockDevice { } fn flush(&mut self) -> ax_driver::prelude::DevResult { - // Intentionally a no-op. ext4 calls this from inside SpinNoPreempt - // where sleeping VFS I/O would panic. Dirty data is written back in - // LOOP_CLR_FD (losetup -d), BLKFLSBUF, or after umount — all in - // normal syscall context. + // Intentionally a no-op. Filesystem block I/O paths must not re-enter + // backing-file VFS writeback. Dirty data is written back in LOOP_CLR_FD + // (losetup -d), BLKFLSBUF, or after umount, all in normal syscall + // context. Ok(()) } } @@ -499,8 +499,8 @@ impl DeviceOps for LoopDevice { // Write back dirty data from the block cache before clearing. // This runs in normal syscall context so CachedFile VFS I/O - // (page cache updates) is safe. The SpinNoPreempt lock - // ensures no concurrent write_block() can race. + // (page cache updates) is safe. The cache lock ensures no + // concurrent write_block() can race. #[cfg(feature = "ext4")] { let cache = self.block_cache.lock(); diff --git a/os/StarryOS/kernel/src/pseudofs/dev/pwm.rs b/os/StarryOS/kernel/src/pseudofs/dev/pwm.rs index b2b6478806..0d71c9ff1d 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/pwm.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/pwm.rs @@ -2,11 +2,11 @@ use alloc::{borrow::Cow, boxed::Box, format, sync::Arc, vec::Vec}; use ax_sync::Mutex; use axfs_ng_vfs::{VfsError, VfsResult}; -use lazy_static::lazy_static; use sg200x_bsp::{ pwm::{Pwm, PwmChannel, PwmMode, PwmPolarity}, soc::PWM0_BASE, }; +use spin::Lazy; use crate::pseudofs::{ DirMaker, NodeOpsMux, RwFile, SimpleDir, SimpleDirOps, SimpleFile, SimpleFileOperation, @@ -60,9 +60,7 @@ impl PwmSysfsState { } } -lazy_static! { - static ref PWM_SYSFS_STATE: Mutex = Mutex::new(PwmSysfsState::new()); -} +static PWM_SYSFS_STATE: Lazy> = Lazy::new(|| Mutex::new(PwmSysfsState::new())); struct PwmClassDir { fs: Arc, diff --git a/os/StarryOS/kernel/src/pseudofs/dev/tty/mod.rs b/os/StarryOS/kernel/src/pseudofs/dev/tty/mod.rs index f037e3b634..f903296030 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/tty/mod.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/tty/mod.rs @@ -115,7 +115,7 @@ impl DeviceOps for Tty { } TCSETS | TCSETSF | TCSETSW => { // TODO: drain output? - // Note: vm_read() must complete before acquiring the SpinNoPreempt lock. + // Note: vm_read() must complete before acquiring the terminal lock. // Faultable user memory access inside an atomic context (preemption // disabled) will call might_sleep() in handle_page_fault and panic. let termios = Arc::new(Termios2::new((arg as *const Termios).vm_read()?)); diff --git a/os/StarryOS/kernel/src/pseudofs/dev/tty/ntty.rs b/os/StarryOS/kernel/src/pseudofs/dev/tty/ntty.rs index 420c1ecf09..75106cf3e7 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/tty/ntty.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/tty/ntty.rs @@ -1,7 +1,7 @@ use alloc::sync::Arc; use axpoll::PollSet; -use lazy_static::lazy_static; +use spin::Lazy; use super::{ Tty, @@ -23,11 +23,9 @@ impl TtyWrite for Console { } } -lazy_static! { - /// The default TTY device. - pub static ref N_TTY: Arc = new_n_tty(); - static ref CONSOLE_INPUT_SOURCE: Arc = Arc::new(PollSet::new()); -} +/// The default TTY device. +pub static N_TTY: Lazy> = Lazy::new(new_n_tty); +static CONSOLE_INPUT_SOURCE: Lazy> = Lazy::new(|| Arc::new(PollSet::new())); fn handle_console_input_irq(_irq_num: usize) { let events = ax_hal::console::handle_irq(); diff --git a/os/StarryOS/kernel/src/pseudofs/dev/tty/pty.rs b/os/StarryOS/kernel/src/pseudofs/dev/tty/pty.rs index cf8a5784ca..5fecba3fda 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/tty/pty.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/tty/pty.rs @@ -1,6 +1,6 @@ use alloc::sync::Arc; -use ax_kspin::SpinNoPreempt; +use ax_kspin::SpinNoIrq; use axpoll::PollSet; use ringbuf::{ Cons, HeapRb, Prod, @@ -36,11 +36,11 @@ impl TtyRead for PtyReader { } #[derive(Clone)] -pub struct PtyWriter(Arc>>, Arc); +pub struct PtyWriter(Arc>>, Arc); impl PtyWriter { pub fn new(buffer: Buffer, poll_rx: Arc) -> Self { - Self(Arc::new(SpinNoPreempt::new(Prod::new(buffer))), poll_rx) + Self(Arc::new(SpinNoIrq::new(Prod::new(buffer))), poll_rx) } } diff --git a/os/StarryOS/kernel/src/pseudofs/dev/tty/terminal/mod.rs b/os/StarryOS/kernel/src/pseudofs/dev/tty/terminal/mod.rs index 9af9b57fe2..b37ad5dcc6 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/tty/terminal/mod.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/tty/terminal/mod.rs @@ -3,7 +3,7 @@ use alloc::sync::Arc; use core::sync::atomic::AtomicU32; -use ax_kspin::SpinNoPreempt; +use ax_kspin::SpinNoIrq; use bytemuck::AnyBitPattern; pub mod job; @@ -21,21 +21,21 @@ pub struct WindowSize { pub struct Terminal { pub job_control: job::JobControl, - pub window_size: SpinNoPreempt, - pub termios: SpinNoPreempt>, + pub window_size: SpinNoIrq, + pub termios: SpinNoIrq>, pub pty_number: AtomicU32, } impl Default for Terminal { fn default() -> Self { Self { job_control: job::JobControl::new(), - window_size: SpinNoPreempt::new(WindowSize { + window_size: SpinNoIrq::new(WindowSize { ws_row: 28, ws_col: 110, ws_xpixel: 0, ws_ypixel: 0, }), - termios: SpinNoPreempt::new(Arc::new(termios::Termios2::default())), + termios: SpinNoIrq::new(Arc::new(termios::Termios2::default())), pty_number: AtomicU32::new(0), } } diff --git a/os/StarryOS/kernel/src/pseudofs/tmp.rs b/os/StarryOS/kernel/src/pseudofs/tmp.rs index a65421889e..ba6c3ba27d 100644 --- a/os/StarryOS/kernel/src/pseudofs/tmp.rs +++ b/os/StarryOS/kernel/src/pseudofs/tmp.rs @@ -2,7 +2,7 @@ use alloc::{borrow::ToOwned, string::String, sync::Arc}; use core::{any::Any, borrow::Borrow, cmp::Ordering, task::Context, time::Duration}; use ax_kspin::SpinNoIrq; -use ax_sync::{LockdepMutexExt, Mutex}; +use ax_sync::Mutex; use axfs_ng_vfs::{ DeviceId, DirEntry, DirEntrySink, DirNode, DirNodeOps, FileNode, FileNodeOps, Filesystem, FilesystemOps, Metadata, MetadataUpdate, NodeFlags, NodeOps, NodePermission, NodeType, @@ -14,8 +14,6 @@ use slab::Slab; use crate::pseudofs::dummy_stat_fs; -const TMPFS_DIR_ENTRIES_NESTED_SUBCLASS: ax_sync::LockSubclass = 1; - #[derive(PartialEq, Eq, Hash, Clone)] struct FileName(String); @@ -47,11 +45,6 @@ where } } -#[inline(always)] -fn lock_tmpfs_nested(mutex: &Mutex) -> ax_sync::MutexGuard<'_, T> { - mutex.lock_nested(TMPFS_DIR_ENTRIES_NESTED_SUBCLASS) -} - impl Borrow for FileName { fn borrow(&self) -> &str { &self.0 @@ -63,7 +56,9 @@ pub struct MemoryFs { // Inodes may be released from atomic cleanup paths, so the slab and // metadata locks must not sleep. inodes: SpinNoIrq>>, - root: Mutex>, + // root_dir() is used while mounting pseudofs during early startup, before + // Starry has reached a sleepable task context. + root: SpinNoIrq>, } impl MemoryFs { @@ -80,14 +75,13 @@ impl MemoryFs { pub fn new_with_handle() -> (Filesystem, Arc) { let handle = Arc::new(Self { inodes: SpinNoIrq::new(Slab::new()), - root: Mutex::default(), + root: SpinNoIrq::new(None), }); let root_ino = Inode::new( &handle, None, NodeType::Directory, NodePermission::from_bits_truncate(0o755), - false, ); *handle.root.lock() = Some(DirEntry::new_dir( |this| DirNode::new(MemoryNode::new(handle.clone(), root_ino, Some(this))), @@ -105,7 +99,7 @@ impl MemoryFs { /// The returned entry is not inserted into any directory, so it has no /// path-based lookup and is kept alive solely by the returned handle(s). pub fn create_anonymous_file(self: &Arc, name: &str, perm: NodePermission) -> DirEntry { - let inode = Inode::new(self, None, NodeType::RegularFile, perm, false); + let inode = Inode::new(self, None, NodeType::RegularFile, perm); DirEntry::new_file( FileNode::new(MemoryNode::new(self.clone(), inode, None)), NodeType::RegularFile, @@ -147,9 +141,19 @@ struct FileContent { symlink: Mutex>, } -#[derive(Default)] struct DirContent { - entries: Mutex>, + // VFS dentry-cache operations call tmpfs directory ops while holding + // SpinNoIrq guards, so this per-directory map must not use a blocking + // mutex. + entries: SpinNoIrq>, +} + +impl Default for DirContent { + fn default() -> Self { + Self { + entries: SpinNoIrq::new(HashMap::new()), + } + } } enum NodeContent { @@ -169,7 +173,6 @@ impl Inode { parent: Option, node_type: NodeType, permission: NodePermission, - nested_dir_entries: bool, ) -> Arc { let mut inodes = fs.inodes.lock(); let entry = inodes.vacant_entry(); @@ -204,11 +207,7 @@ impl Inode { entry.insert(result.clone()); drop(inodes); if let NodeContent::Dir(dir) = &result.content { - let mut entries = if nested_dir_entries { - lock_tmpfs_nested(&dir.entries) - } else { - dir.entries.lock() - }; + let mut entries = dir.entries.lock(); entries.insert(".".into(), InodeRef::new(fs.clone(), ino)); entries.insert( "..".into(), @@ -439,7 +438,7 @@ impl DirNodeOps for MemoryNode { if entries.contains_key(name) { return Err(VfsError::AlreadyExists); } - let inode = Inode::new(&self.fs, Some(self.inode.ino), node_type, permission, true); + let inode = Inode::new(&self.fs, Some(self.inode.ino), node_type, permission); entries.insert(name.into(), InodeRef::new(self.fs.clone(), inode.ino)); self.new_entry(name, node_type, inode) } @@ -454,7 +453,7 @@ impl DirNodeOps for MemoryNode { return Err(VfsError::AlreadyExists); } let inode = target.inode.clone(); - let node_type = target.metadata()?.node_type; + let node_type = inode.metadata.lock().node_type; entries.insert(name.into(), InodeRef::new(self.fs.clone(), inode.ino)); self.new_entry(name, node_type, inode) } @@ -469,7 +468,7 @@ impl DirNodeOps for MemoryNode { }; let inode = entry.get(); if let NodeContent::Dir(DirContent { entries }) = &inode.content - && lock_tmpfs_nested(entries).len() > 2 + && entries.lock().len() > 2 { return Err(VfsError::DirectoryNotEmpty); } diff --git a/os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs b/os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs index c269fdebdd..e183e752ab 100644 --- a/os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs +++ b/os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs @@ -6,6 +6,8 @@ use core::{ }; use ax_errno::{AxError, AxResult, LinuxError}; +use ax_kspin::SpinNoIrq as Mutex; +use ax_sync::Mutex as BlockingMutex; use crab_usb::{ Device, DeviceInfo, Endpoint, EventHandler, ProbedDevice, usb_if::{ @@ -17,7 +19,7 @@ use crab_usb::{ }; use event_listener::{Event as NotifyEvent, listener}; use rdrive::DeviceId as RDriveDeviceId; -use spin::{Mutex, RwLock}; +use spin::RwLock; use starry_vm::{VmMutPtr, vm_load, vm_write_slice}; use super::{ @@ -74,7 +76,7 @@ struct UsbDeviceRecord { type EndpointHandle = Arc>; struct LiveDeviceState { - device: Mutex, + device: BlockingMutex, endpoints: RwLock>, endpoint_interfaces: RwLock>, interface_owners: Mutex>, @@ -227,7 +229,7 @@ fn wait_control( pub(super) struct UsbFsManager { state: Mutex, - open_lock: Mutex<()>, + open_lock: BlockingMutex<()>, pub(super) refresh_event: NotifyEvent, usb_activity: UsbActivity, } @@ -366,7 +368,7 @@ impl UsbFsManager { Self { state: Mutex::new(UsbFsState { hosts, devices }), - open_lock: Mutex::new(()), + open_lock: BlockingMutex::new(()), refresh_event: NotifyEvent::new(), usb_activity: UsbActivity::new(), } @@ -681,7 +683,7 @@ impl UsbFsManager { let mut state = self.state.lock(); let record = state.devices.get_mut(&stable_id).ok_or(AxError::NotFound)?; record.live_device = Some(Arc::new(LiveDeviceState { - device: Mutex::new(live_device), + device: BlockingMutex::new(live_device), endpoints: RwLock::new(BTreeMap::new()), endpoint_interfaces: RwLock::new(BTreeMap::new()), interface_owners: Mutex::new(BTreeMap::new()), diff --git a/os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs b/os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs index aac46bc50f..39302026c9 100644 --- a/os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs +++ b/os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs @@ -18,12 +18,12 @@ use core::{ }; use ax_errno::{AxError, AxResult, LinuxError, LinuxResult}; +use ax_kspin::SpinNoIrq as Mutex; use ax_sync::Mutex as BlockingMutex; use axfs_ng_vfs::Filesystem; use axpoll::{IoEvents, PollSet, Pollable}; use crab_usb::usb_if::endpoint::{TransferCompletion, TransferRequest}; use event_listener::Event as NotifyEvent; -use spin::Mutex; use starry_vm::{VmMutPtr, VmPtr, vm_load, vm_write_slice}; use self::{irq::manager, manager::UsbFsManager, tree::UsbRootDir}; diff --git a/os/StarryOS/kernel/src/syscall/fs/mount.rs b/os/StarryOS/kernel/src/syscall/fs/mount.rs index 1b2784fb74..b7b0f1ee02 100644 --- a/os/StarryOS/kernel/src/syscall/fs/mount.rs +++ b/os/StarryOS/kernel/src/syscall/fs/mount.rs @@ -205,9 +205,9 @@ pub fn sys_umount2(target: *const c_char, flags: i32) -> AxResult { target.unmount()?; - // After unmount the SpinNoPreempt lock inside ext4 is released; safe - // to do VFS I/O here. Propagate writeback errors so userspace sees - // EIO when dirty data could not be persisted to the backing file. + // After unmount, filesystem block I/O has stopped; it is safe to do VFS + // writeback here. Propagate writeback errors so userspace sees EIO when + // dirty data could not be persisted to the backing file. if let Some(cb) = writeback { cb()?; } diff --git a/os/StarryOS/kernel/src/syscall/sys.rs b/os/StarryOS/kernel/src/syscall/sys.rs index 8df846605a..a8a3499e07 100644 --- a/os/StarryOS/kernel/src/syscall/sys.rs +++ b/os/StarryOS/kernel/src/syscall/sys.rs @@ -95,9 +95,8 @@ impl SyslogState { } } -lazy_static::lazy_static! { - static ref SYSLOG_STATE: Mutex = Mutex::new(SyslogState::new()); -} +static SYSLOG_STATE: spin::Lazy> = + spin::Lazy::new(|| Mutex::new(SyslogState::new())); /// Mirror of Linux kernel `uid_valid()` / `make_kuid()` rejection: any caller- /// supplied UID/GID of `(uid_t)-1` (`u32::MAX`) is invalid outside the NOCHG diff --git a/os/StarryOS/kernel/src/task/posix_timer.rs b/os/StarryOS/kernel/src/task/posix_timer.rs index 830c95c17b..6fd711eb1e 100644 --- a/os/StarryOS/kernel/src/task/posix_timer.rs +++ b/os/StarryOS/kernel/src/task/posix_timer.rs @@ -8,12 +8,12 @@ use core::{ use ax_errno::{AxError, AxResult}; use ax_hal::time::{NANOS_PER_SEC, monotonic_time_nanos, wall_time}; +use ax_kspin::SpinNoIrq as Mutex; use linux_raw_sys::general::{ CLOCK_BOOTTIME, CLOCK_MONOTONIC, CLOCK_MONOTONIC_COARSE, CLOCK_MONOTONIC_RAW, CLOCK_PROCESS_CPUTIME_ID, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, CLOCK_THREAD_CPUTIME_ID, SIGEV_NONE, SIGEV_SIGNAL, }; -use spin::Mutex; use starry_process::Pid; use starry_signal::{SignalInfo, Signo}; diff --git a/os/StarryOS/kernel/src/task/timer.rs b/os/StarryOS/kernel/src/task/timer.rs index f537347532..2f5dfc8b68 100644 --- a/os/StarryOS/kernel/src/task/timer.rs +++ b/os/StarryOS/kernel/src/task/timer.rs @@ -4,13 +4,13 @@ use alloc::{borrow::ToOwned, collections::binary_heap::BinaryHeap, sync::Arc}; use core::{mem, time::Duration}; use ax_hal::time::{NANOS_PER_SEC, TimeValue, monotonic_time_nanos, wall_time}; +use ax_kspin::SpinNoIrq as Mutex; use ax_task::{ WeakAxTaskRef, current, future::{block_on, timeout_at}, }; use event_listener::{Event, listener}; -use lazy_static::lazy_static; -use spin::Mutex; +use spin::Lazy; use starry_process::Pid; use starry_signal::Signo; use strum::FromRepr; @@ -51,10 +51,8 @@ impl Ord for Entry { } } -lazy_static! { - static ref ALARM_LIST: Mutex> = Mutex::new(BinaryHeap::new()); - static ref EVENT_NEW_TIMER: Event = Event::new(); -} +static ALARM_LIST: Lazy>> = Lazy::new(|| Mutex::new(BinaryHeap::new())); +static EVENT_NEW_TIMER: Lazy = Lazy::new(Event::new); /// The type of interval timer. #[repr(i32)] diff --git a/os/arceos/api/arceos_posix_api/Cargo.toml b/os/arceos/api/arceos_posix_api/Cargo.toml index f3e50a67f2..73efef989e 100644 --- a/os/arceos/api/arceos_posix_api/Cargo.toml +++ b/os/arceos/api/arceos_posix_api/Cargo.toml @@ -50,7 +50,6 @@ ax-task = { workspace = true, optional = true } ax-errno.workspace = true ax-io.workspace = true flatten_objects = "0.2" -lazy_static.workspace = true scope-local = { workspace = true, optional = true } spin.workspace = true diff --git a/os/arceos/api/arceos_posix_api/src/imp/pthread/mod.rs b/os/arceos/api/arceos_posix_api/src/imp/pthread/mod.rs index a9cc77ead5..4140e8e33b 100644 --- a/os/arceos/api/arceos_posix_api/src/imp/pthread/mod.rs +++ b/os/arceos/api/arceos_posix_api/src/imp/pthread/mod.rs @@ -6,14 +6,14 @@ use core::{ use ax_errno::{LinuxError, LinuxResult}; use ax_task::AxTaskRef; -use spin::RwLock; +use spin::{Lazy, RwLock}; use crate::ctypes; pub mod mutex; -lazy_static::lazy_static! { - static ref TID_TO_PTHREAD: RwLock>> = { +static TID_TO_PTHREAD: Lazy>>> = + Lazy::new(|| { let mut map = BTreeMap::new(); let main_task = ax_task::current(); let main_tid = main_task.id().as_u64(); @@ -26,8 +26,7 @@ lazy_static::lazy_static! { let ptr = Box::into_raw(Box::new(main_thread)) as *mut c_void; map.insert(main_tid, ForceSendSync(ptr)); RwLock::new(map) - }; -} + }); struct Packet { result: UnsafeCell, diff --git a/os/arceos/modules/axfs-ng/src/fs/ext4/lwext4/fs.rs b/os/arceos/modules/axfs-ng/src/fs/ext4/lwext4/fs.rs index 9c4c062463..c54d2d6def 100644 --- a/os/arceos/modules/axfs-ng/src/fs/ext4/lwext4/fs.rs +++ b/os/arceos/modules/axfs-ng/src/fs/ext4/lwext4/fs.rs @@ -2,7 +2,7 @@ use alloc::sync::Arc; use core::cell::OnceCell; use ax_driver::{AxBlockDevice, PartitionRegion}; -use ax_kspin::{SpinNoPreempt as Mutex, SpinNoPreemptGuard as MutexGuard}; +use ax_kspin::{SpinNoIrq as Mutex, SpinNoIrqGuard as MutexGuard}; use axfs_ng_vfs::{ DirEntry, DirNode, Filesystem, FilesystemOps, Reference, StatFs, VfsResult, path::MAX_NAME_LEN, }; @@ -36,6 +36,13 @@ impl Ext4Filesystem { Ok(Filesystem::new(fs)) } + /// Locks the shared lwext4 state. + /// + /// lwext4 operations may call into the block device while this guard is + /// held. The current rootfs setup can also run in early atomic contexts + /// where a blocking mutex trips `might_sleep()`, so use `SpinNoIrq` + /// instead of the older `SpinNoPreempt` to close same-CPU IRQ reentry + /// without changing the boot-time calling contract. pub(crate) fn lock(&self) -> MutexGuard<'_, LwExt4Filesystem> { self.inner.lock() } diff --git a/os/arceos/modules/axfs-ng/src/fs/ext4/rsext4/fs.rs b/os/arceos/modules/axfs-ng/src/fs/ext4/rsext4/fs.rs index 7f74e5929f..0c14a1ca89 100644 --- a/os/arceos/modules/axfs-ng/src/fs/ext4/rsext4/fs.rs +++ b/os/arceos/modules/axfs-ng/src/fs/ext4/rsext4/fs.rs @@ -2,7 +2,7 @@ use alloc::{boxed::Box, sync::Arc}; use core::cell::OnceCell; use ax_driver::{AxBlockDevice, PartitionRegion, prelude::BlockDriverOps}; -use ax_kspin::{SpinNoPreempt as Mutex, SpinNoPreemptGuard as MutexGuard}; +use ax_kspin::{SpinNoIrq as Mutex, SpinNoIrqGuard as MutexGuard}; use axfs_ng_vfs::{ DirEntry, DirNode, Filesystem, FilesystemOps, Reference, StatFs, VfsResult, path::MAX_NAME_LEN, }; @@ -62,6 +62,14 @@ impl Ext4Filesystem { Ok(Filesystem::new(fs)) } + /// Locks the shared rsext4 state. + /// + /// rsext4 operations may allocate, flush caches, commit journal state, and + /// call into the block device while this guard is held. The current rootfs + /// setup can also run in early atomic contexts where a blocking mutex trips + /// `might_sleep()`, so use `SpinNoIrq` instead of the older + /// `SpinNoPreempt` to close same-CPU IRQ reentry without changing the + /// boot-time calling contract. pub(crate) fn lock(&self) -> MutexGuard<'_, Ext4State> { self.inner.lock() } diff --git a/os/arceos/modules/axfs-ng/src/fs/fat/fs.rs b/os/arceos/modules/axfs-ng/src/fs/fat/fs.rs index 9f42948372..6c05965237 100644 --- a/os/arceos/modules/axfs-ng/src/fs/fat/fs.rs +++ b/os/arceos/modules/axfs-ng/src/fs/fat/fs.rs @@ -2,7 +2,7 @@ use alloc::sync::Arc; use core::marker::PhantomPinned; use ax_driver::{AxBlockDevice, PartitionRegion}; -use ax_kspin::{SpinNoPreempt as Mutex, SpinNoPreemptGuard as MutexGuard}; +use ax_kspin::{SpinNoIrq as Mutex, SpinNoIrqGuard as MutexGuard}; use axfs_ng_vfs::{ DirEntry, Filesystem, FilesystemOps, Reference, StatFs, VfsResult, path::MAX_NAME_LEN, }; @@ -63,6 +63,13 @@ impl FatFilesystem { } impl FatFilesystem { + /// Locks the shared FAT state. + /// + /// FAT operations may perform block I/O while this guard is held. The + /// current rootfs setup can also run in early atomic contexts where a + /// blocking mutex trips `might_sleep()`, so use `SpinNoIrq` instead of the + /// older `SpinNoPreempt` to close same-CPU IRQ reentry without changing the + /// boot-time calling contract. pub(crate) fn lock(&self) -> MutexGuard<'_, FatFilesystemInner> { self.inner.lock() } diff --git a/os/arceos/modules/axfs-ng/src/highlevel/fs.rs b/os/arceos/modules/axfs-ng/src/highlevel/fs.rs index f5fb6cd765..e9b424e792 100644 --- a/os/arceos/modules/axfs-ng/src/highlevel/fs.rs +++ b/os/arceos/modules/axfs-ng/src/highlevel/fs.rs @@ -7,6 +7,7 @@ use alloc::{ }; use ax_io::{Read, Write}; +use ax_kspin::SpinNoIrq; use ax_sync::Mutex; use axfs_ng_vfs::{ Location, Metadata, Mountpoint, NodePermission, NodeType, VfsError, VfsResult, @@ -29,7 +30,7 @@ pub static ROOT_FS_CONTEXT: Once = Once::new(); /// [`FsContext::propagate_pivot_root`] to iterate over every task's /// filesystem context and apply the same root / cwd fixup that Linux /// performs in `chroot_fs_refs()` after `pivot_root(2)`. -static FS_REGISTRY: spin::Mutex>>> = spin::Mutex::new(Vec::new()); +static FS_REGISTRY: SpinNoIrq>>> = SpinNoIrq::new(Vec::new()); /// Register an `FsContext` in the global [`FS_REGISTRY`]. fn register_fs_context(ctx: &Arc>) { diff --git a/os/arceos/modules/axfs/Cargo.toml b/os/arceos/modules/axfs/Cargo.toml index abeb8af3b8..5ff4cf82ce 100644 --- a/os/arceos/modules/axfs/Cargo.toml +++ b/os/arceos/modules/axfs/Cargo.toml @@ -21,6 +21,7 @@ ax-fs-ramfs = { workspace = true } ax-fs-vfs = { workspace = true } ax-io = { workspace = true, features = ["alloc"] } ax-cap-access = { workspace = true } +ax-kspin = { workspace = true } ax-lazyinit = { workspace = true } log = { workspace = true } rsext4 = { workspace = true } diff --git a/os/arceos/modules/axfs/src/dev.rs b/os/arceos/modules/axfs/src/dev.rs index 3f14483a96..9b97e49b8f 100644 --- a/os/arceos/modules/axfs/src/dev.rs +++ b/os/arceos/modules/axfs/src/dev.rs @@ -17,7 +17,7 @@ use alloc::sync::Arc; use ax_driver::prelude::*; -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; const BLOCK_SIZE: usize = 512; diff --git a/os/arceos/modules/axfs/src/fs/ext4fs.rs b/os/arceos/modules/axfs/src/fs/ext4fs.rs index 19ed79e7ac..f0ea2117b3 100644 --- a/os/arceos/modules/axfs/src/fs/ext4fs.rs +++ b/os/arceos/modules/axfs/src/fs/ext4fs.rs @@ -24,6 +24,7 @@ use ax_fs_vfs::{ VfsDirEntry, VfsError, VfsNodeAttr, VfsNodeOps, VfsNodePerm, VfsNodeRef, VfsNodeType, VfsOps, VfsResult, }; +use ax_kspin::SpinNoIrq as Mutex; use rsext4::{ Ext4Error, Ext4FileSystem as Rsext4FileSystem, Ext4Result, Ext4Timestamp, Jbd2Dev, api::{OpenFile, fs_mount, lseek, open, read_at}, @@ -32,7 +33,6 @@ use rsext4::{ file::{delete_dir, is_dir_empty, mkfile, mv, truncate, unlink, write_file}, loopfile::resolve_inode_block_allextend, }; -use spin::Mutex; use crate::dev::{Disk, Partition}; diff --git a/os/arceos/modules/axfs/src/fs/fatfs.rs b/os/arceos/modules/axfs/src/fs/fatfs.rs index 20d3d7baa4..7294737fee 100644 --- a/os/arceos/modules/axfs/src/fs/fatfs.rs +++ b/os/arceos/modules/axfs/src/fs/fatfs.rs @@ -21,8 +21,8 @@ use ax_fs_vfs::{ VfsDirEntry, VfsError, VfsNodeAttr, VfsNodeOps, VfsNodePerm, VfsNodeRef, VfsNodeType, VfsOps, VfsResult, }; +use ax_kspin::SpinNoIrq as Mutex; use axfatfs::{Dir, File, LossyOemCpConverter, NullTimeProvider, Read, Seek, SeekFrom, Write}; -use spin::Mutex; use crate::dev::{Disk, Partition}; diff --git a/os/arceos/modules/axfs/src/root.rs b/os/arceos/modules/axfs/src/root.rs index d58347c4bb..5ba0e364d9 100644 --- a/os/arceos/modules/axfs/src/root.rs +++ b/os/arceos/modules/axfs/src/root.rs @@ -27,8 +27,8 @@ use alloc::{ use ax_errno::{AxError, AxResult}; use ax_fs_vfs::{VfsDirEntry, VfsNodeAttr, VfsNodeOps, VfsNodeRef, VfsNodeType, VfsOps, VfsResult}; +use ax_kspin::SpinNoIrq as Mutex; use ax_lazyinit::LazyInit; -use spin::Mutex; use crate::{ api::FileType, diff --git a/os/arceos/modules/axnet-ng/Cargo.toml b/os/arceos/modules/axnet-ng/Cargo.toml index 2769653c95..bd511a472a 100644 --- a/os/arceos/modules/axnet-ng/Cargo.toml +++ b/os/arceos/modules/axnet-ng/Cargo.toml @@ -20,6 +20,7 @@ ax-fs-ng = { workspace = true } axfs-ng-vfs = { workspace = true } ax-hal = { workspace = true } ax-io = { workspace = true } +ax-kspin = { workspace = true } axpoll = { workspace = true } ax-sync = { workspace = true } ax-task = { workspace = true, features = ["irq", "multitask"] } @@ -28,7 +29,6 @@ cfg-if = { workspace = true } enum_dispatch = "0.3" event-listener = { version = "5.4", default-features = false } hashbrown = "0.16" -lazy_static = { workspace = true } log = { workspace = true } ringbuf = { version = "0.4", default-features = false, features = ["alloc"] } spin = { workspace = true } diff --git a/os/arceos/modules/axnet-ng/src/raw.rs b/os/arceos/modules/axnet-ng/src/raw.rs index b232148b25..ede746db3b 100644 --- a/os/arceos/modules/axnet-ng/src/raw.rs +++ b/os/arceos/modules/axnet-ng/src/raw.rs @@ -13,6 +13,7 @@ use core::{ use ax_errno::{AxError, AxResult, LinuxError, ax_bail}; use ax_io::prelude::*; +use ax_kspin::SpinNoIrq as Mutex; use axpoll::{IoEvents, Pollable}; pub use smoltcp::wire::{IpProtocol, IpVersion}; use smoltcp::{ @@ -21,7 +22,7 @@ use smoltcp::{ storage::PacketMetadata, wire::{Icmpv6Packet, IpAddress, IpListenEndpoint, Ipv4Packet, Ipv4Repr, Ipv6Packet, Ipv6Repr}, }; -use spin::{Mutex, RwLock}; +use spin::RwLock; use crate::{ RecvFlags, RecvOptions, SOCKET_SET, SendFlags, SendOptions, Shutdown, SocketAddrEx, SocketOps, diff --git a/os/arceos/modules/axnet-ng/src/unix/mod.rs b/os/arceos/modules/axnet-ng/src/unix/mod.rs index 8ad094cf13..d2a0165a3a 100644 --- a/os/arceos/modules/axnet-ng/src/unix/mod.rs +++ b/os/arceos/modules/axnet-ng/src/unix/mod.rs @@ -14,7 +14,7 @@ use axfs_ng_vfs::NodeType; use axpoll::{IoEvents, Pollable}; use enum_dispatch::enum_dispatch; use hashbrown::HashMap; -use lazy_static::lazy_static; +use spin::Lazy; pub use self::{dgram::DgramTransport, stream::StreamTransport}; use crate::{ @@ -93,9 +93,8 @@ pub struct BindSlot { dgram: Mutex>, } -lazy_static! { - static ref ABSTRACT_BINDS: Mutex, BindSlot>> = Mutex::new(HashMap::new()); -} +static ABSTRACT_BINDS: Lazy, BindSlot>>> = + Lazy::new(|| Mutex::new(HashMap::new())); pub(crate) fn with_slot( addr: &UnixSocketAddr, @@ -116,11 +115,16 @@ pub(crate) fn with_slot( if loc.metadata()?.node_type != NodeType::Socket { return Err(AxError::NotASocket); } - f(loc - .user_data() - .get::() - .ok_or(AxError::ConnectionRefused)? - .as_ref()) + let slot = { + // `DirEntry::user_data()` is protected by a SpinNoIrq guard. + // Drop it before running transport code, which may take + // sleepable socket mutexes. + let user_data = loc.user_data(); + user_data + .get::() + .ok_or(AxError::ConnectionRefused)? + }; + f(slot.as_ref()) } } } @@ -144,10 +148,14 @@ fn with_slot_or_insert( if loc.metadata()?.node_type != NodeType::Socket { return Err(AxError::NotASocket); } - f(loc - .user_data() - .get_or_insert_with(BindSlot::default) - .as_ref()) + let slot = { + // `DirEntry::user_data()` is protected by a SpinNoIrq guard. + // Drop it before running transport code, which may take + // sleepable socket mutexes. + let mut user_data = loc.user_data(); + user_data.get_or_insert_with(BindSlot::default) + }; + f(slot.as_ref()) } } } diff --git a/os/arceos/ulib/arceos-rust/Cargo.toml b/os/arceos/ulib/arceos-rust/Cargo.toml index 6aa0b7103e..9a3d88d545 100644 --- a/os/arceos/ulib/arceos-rust/Cargo.toml +++ b/os/arceos/ulib/arceos-rust/Cargo.toml @@ -42,6 +42,7 @@ tls = [] # Multi-threading and scheduler multitask = [] +lockdep = [] sched-fifo = [] sched-rr = [] sched-cfs = [] diff --git a/os/arceos/ulib/arceos-rust/lib/Cargo.toml b/os/arceos/ulib/arceos-rust/lib/Cargo.toml index 72778bdbe8..022fd06475 100644 --- a/os/arceos/ulib/arceos-rust/lib/Cargo.toml +++ b/os/arceos/ulib/arceos-rust/lib/Cargo.toml @@ -49,6 +49,7 @@ tls = ["ax-feat/tls"] # Multi-threading and scheduler multitask = ["ax-api/multitask", "ax-feat/multitask", "irq"] +lockdep = ["ax-feat/lockdep"] sched-cfs = ["ax-feat/sched-cfs"] sched-fifo = ["ax-feat/sched-fifo"] sched-rr = ["ax-feat/sched-rr"] diff --git a/os/axvisor/Cargo.toml b/os/axvisor/Cargo.toml index c4bf19a40f..345b078aa3 100644 --- a/os/axvisor/Cargo.toml +++ b/os/axvisor/Cargo.toml @@ -50,16 +50,17 @@ sdmmc = [ rockchip-pm = ["dep:axplat-dyn", "axplat-dyn/rockchip-pm"] serial = ["dep:axplat-dyn", "axplat-dyn/serial"] +[dependencies] +spin = { workspace = true } + [target.'cfg(not(any(windows, unix)))'.dependencies] bitflags = "2.2" cfg-if = "1.0" ax-cpumask = { workspace = true } ax-kernel-guard = { workspace = true } ax-kspin = { workspace = true } -lazy_static = {version = "1.5", default-features = false, features = ["spin_no_std"]} ax-lazyinit = { workspace = true } log = "0.4" -spin = "0.10" ax-timer-list = { workspace = true } hashbrown = "0.14" diff --git a/os/axvisor/src/hal/arch/loongarch64/mod.rs b/os/axvisor/src/hal/arch/loongarch64/mod.rs index 43a6b4c08f..cde4f6d61d 100644 --- a/os/axvisor/src/hal/arch/loongarch64/mod.rs +++ b/os/axvisor/src/hal/arch/loongarch64/mod.rs @@ -3,7 +3,7 @@ mod api; pub mod cache; -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; const CSR_GSTAT: u16 = 0x50; const CSR_GINTC: u16 = 0x52; diff --git a/os/axvisor/src/shell/command/mod.rs b/os/axvisor/src/shell/command/mod.rs index 969d8fc671..b6be05f55c 100644 --- a/os/axvisor/src/shell/command/mod.rs +++ b/os/axvisor/src/shell/command/mod.rs @@ -26,9 +26,9 @@ use std::vec::Vec; use std::{collections::BTreeMap, string::ToString}; use std::{print, println}; -lazy_static::lazy_static! { - pub static ref COMMAND_TREE: BTreeMap = build_command_tree(); -} +use spin::Lazy; + +pub static COMMAND_TREE: Lazy> = Lazy::new(build_command_tree); #[derive(Debug, Clone)] pub struct CommandNode { diff --git a/os/axvisor/src/vmm/fdt/mod.rs b/os/axvisor/src/vmm/fdt/mod.rs index 07cde73152..7bff3d0525 100644 --- a/os/axvisor/src/vmm/fdt/mod.rs +++ b/os/axvisor/src/vmm/fdt/mod.rs @@ -25,10 +25,10 @@ mod vm_fdt; use alloc::collections::BTreeMap; use alloc::vec::Vec; +use ax_kspin::SpinNoIrq as Mutex; use ax_lazyinit::LazyInit; use axvm::config::{AxVMConfig, AxVMCrateConfig}; use fdt_parser::Fdt; -use spin::Mutex; pub use parser::*; // pub use print::print_fdt; diff --git a/os/axvisor/src/vmm/vm_list.rs b/os/axvisor/src/vmm/vm_list.rs index 136a49949e..63811b7f23 100644 --- a/os/axvisor/src/vmm/vm_list.rs +++ b/os/axvisor/src/vmm/vm_list.rs @@ -15,7 +15,7 @@ use alloc::collections::BTreeMap; use alloc::vec::Vec; -use spin::Mutex; +use ax_kspin::SpinNoIrq as Mutex; use crate::vmm::VMRef; diff --git a/platform/axplat-dyn/src/drivers/blk/mod.rs b/platform/axplat-dyn/src/drivers/blk/mod.rs index cd398e4e40..229c79117d 100644 --- a/platform/axplat-dyn/src/drivers/blk/mod.rs +++ b/platform/axplat-dyn/src/drivers/blk/mod.rs @@ -17,13 +17,11 @@ use core::{ use ax_driver_base::{BaseDriverOps, DevError, DevResult, DeviceType}; use ax_driver_block::BlockDriverOps; -#[cfg(feature = "irq")] use ax_kspin::SpinNoIrq; #[cfg(feature = "irq")] use ax_plat::irq; use rd_block::BlkError; use rdrive::Device; -use spin::Mutex; use super::DmaImpl; @@ -39,7 +37,7 @@ pub struct Block { irq_num: Option, #[cfg(feature = "irq")] irq_state: Option>, - queue: Mutex, + queue: SpinNoIrq, } pub struct PlatformBlockDevice { @@ -263,7 +261,7 @@ impl TryFrom> for Block { irq_num, #[cfg(feature = "irq")] irq_state, - queue: Mutex::new(queue), + queue: SpinNoIrq::new(queue), }) } } diff --git a/platform/axplat-dyn/src/drivers/blk/virtio_pci.rs b/platform/axplat-dyn/src/drivers/blk/virtio_pci.rs index b080230a14..d2661642f3 100644 --- a/platform/axplat-dyn/src/drivers/blk/virtio_pci.rs +++ b/platform/axplat-dyn/src/drivers/blk/virtio_pci.rs @@ -6,6 +6,7 @@ use ax_driver_base::DeviceType; use ax_driver_virtio::pci::{ ConfigurationAccess, DeviceFunction, DeviceFunctionInfo, HeaderType, PciRoot, }; +use ax_kspin::SpinNoIrq as Mutex; use rdrive::{ PlatformDevice, module_driver, probe::{ @@ -13,7 +14,6 @@ use rdrive::{ pci::{Endpoint, EndpointRc}, }, }; -use spin::Mutex; use super::virtio::{VirtIoBlkDevice, register_virtio_block}; use crate::drivers::virtio::VirtIoHalImpl; diff --git a/platform/axplat-dyn/src/drivers/mod.rs b/platform/axplat-dyn/src/drivers/mod.rs index c979b19f43..7c45c2c803 100644 --- a/platform/axplat-dyn/src/drivers/mod.rs +++ b/platform/axplat-dyn/src/drivers/mod.rs @@ -7,11 +7,11 @@ use ax_driver_block::BlockDriverOps; #[cfg(feature = "net")] use ax_driver_net::NetDriverOps; use ax_errno::AxError; +use ax_kspin::SpinNoIrq as Mutex; use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr}; use ax_plat::mem::PhysAddr; use heapless::Vec; use rdrive::probe::OnProbeError; -use spin::Mutex; mod pci; #[cfg(feature = "rknpu")] diff --git a/platform/axplat-dyn/src/drivers/net/virtio_pci.rs b/platform/axplat-dyn/src/drivers/net/virtio_pci.rs index cf603c2bc7..fe3a0061b7 100644 --- a/platform/axplat-dyn/src/drivers/net/virtio_pci.rs +++ b/platform/axplat-dyn/src/drivers/net/virtio_pci.rs @@ -6,6 +6,7 @@ use ax_driver_base::DeviceType; use ax_driver_virtio::pci::{ ConfigurationAccess, DeviceFunction, DeviceFunctionInfo, HeaderType, PciRoot, }; +use ax_kspin::SpinNoIrq as Mutex; use rdrive::{ PlatformDevice, module_driver, probe::{ @@ -13,7 +14,6 @@ use rdrive::{ pci::{Endpoint, EndpointRc}, }, }; -use spin::Mutex; use super::PlatformDeviceNetDriver; use crate::drivers::virtio::VirtIoHalImpl; diff --git a/platform/axplat-dyn/src/drivers/pci.rs b/platform/axplat-dyn/src/drivers/pci.rs index d249f1556a..2f08d35d04 100644 --- a/platform/axplat-dyn/src/drivers/pci.rs +++ b/platform/axplat-dyn/src/drivers/pci.rs @@ -2,6 +2,7 @@ extern crate alloc; use alloc::format; +use ax_kspin::SpinNoIrq as Mutex; use fdt_edit::{PciRange, PciSpace}; use heapless::Vec as ArrayVec; use rdrive::{ @@ -9,7 +10,6 @@ use rdrive::{ probe::{OnProbeError, fdt::NodeType, pci::*}, register::FdtInfo, }; -use spin::Mutex; mod rk3588; diff --git a/platform/axplat-dyn/src/drivers/soc/scmi.rs b/platform/axplat-dyn/src/drivers/soc/scmi.rs index 11a649a1b1..d94313c361 100644 --- a/platform/axplat-dyn/src/drivers/soc/scmi.rs +++ b/platform/axplat-dyn/src/drivers/soc/scmi.rs @@ -2,11 +2,11 @@ use alloc::format; use core::sync::atomic::{AtomicBool, Ordering}; use arm_scmi_rs::{Scmi, Shmem, Smc}; +use ax_kspin::SpinNoIrq as Mutex; use fdt_edit::Phandle; use rdrive::{ DriverGeneric, PlatformDevice, module_driver, probe::OnProbeError, register::FdtInfo, }; -use spin::Mutex; use crate::drivers::iomap; diff --git a/reports/external-spin-audit.md b/reports/external-spin-audit.md new file mode 100644 index 0000000000..d972c71b75 --- /dev/null +++ b/reports/external-spin-audit.md @@ -0,0 +1,375 @@ +# External `spin` crate audit + +Date: 2026-05-21 + +This note records the current direct and indirect references to the external +`spin` crate. It is intended to support the lockdep follow-up that migrates +lockdep-relevant kernel locks away from third-party `spin::{Mutex,RwLock}` and +toward project-local primitives such as `ax_kspin`. + +The migration plan based on this audit is recorded in +[`reports/external-spin-migration-plan.md`](external-spin-migration-plan.md). + +## Method + +Two static sources were checked: + +- `Cargo.lock`, to find packages that directly or transitively depend on the + external `spin` crate. +- `Cargo.toml` and Rust sources, to find workspace crates that explicitly + declare or use `spin`. + +No ArceOS or StarryOS build is required for this audit. Building only validates a +specific target/feature combination. For dependency reachability, `Cargo.lock` +and source scanning are broader. If a later step needs per-target/per-feature +reachability, prefer `cargo tree -e features --target ...` over full builds. + +At the time of this audit, `cargo tree` is not a reliable first step in this +workspace because dependency resolution attempts to query the configured +`rsproxy-sparse` index and the local offline index does not contain the locked +`sg200x-bsp = 0.6.0` entry. Parsing `Cargo.lock` avoids that network/index +dependency. + +## `Cargo.lock` entries + +The workspace lockfile contains two external `spin` versions: + +- `spin 0.9.8` +- `spin 0.10.0` + +## Direct `Cargo.lock` dependents + +These packages directly depend on external `spin` according to `Cargo.lock`: + +```text +arm-scmi-rs 0.1.2 -> spin 0.10.0 +arm_vcpu 0.5.8 -> spin 0.10.0 +arm_vgic 0.4.9 -> spin 0.10.0 +ax-driver-net 0.3.13 -> spin 0.9.8 +ax-fs 0.5.13 -> spin 0.10.0 +ax-fs-devfs 0.3.10 -> spin 0.9.8 +ax-fs-ng 0.5.14 -> spin 0.10.0 +ax-fs-ramfs 0.3.11 -> spin 0.9.8 +ax-hal 0.5.14 -> spin 0.10.0 +ax-net 0.5.13 -> spin 0.10.0 +ax-net-ng 0.6.0 -> spin 0.10.0 +ax-percpu 0.4.11 -> spin 0.10.0 +ax-plat-aarch64-peripherals 0.5.9 -> spin 0.10.0 +ax-posix-api 0.5.15 -> spin 0.10.0 +ax-std 0.5.14 -> spin 0.10.0 +ax-task 0.5.15 -> spin 0.10.0 +axaddrspace 0.5.10 -> spin 0.10.0 +axbacktrace 0.3.9 -> spin 0.10.0 +axdevice 0.4.9 -> spin 0.10.0 +axfs-ng-vfs 0.4.1 -> spin 0.10.0 +axplat-dyn 0.6.1 -> spin 0.10.0 +axpoll 0.3.9 -> spin 0.10.0 +axvisor 0.5.7 -> spin 0.10.0 +axvm 0.5.8 -> spin 0.10.0 +buddy-slab-allocator 0.4.0 -> spin 0.10.0 +buddy_system_allocator 0.12.0 -> spin 0.10.0 +crab-usb 0.9.3 -> spin 0.10.0 +dma-api 0.7.3 -> spin 0.10.0 +lazy_static 1.5.0 -> spin 0.9.8 +loongarch_vcpu 0.5.2 -> spin 0.10.0 +nvme-driver 0.4.2 -> spin 0.10.0 +ramdisk 0.1.1 -> spin 0.10.0 +rdif-serial 0.7.1 -> spin 0.10.0 +rdrive 0.20.1 -> spin 0.10.0 +realtek-rtl8125 0.2.0 -> spin 0.10.0 +riscv_vplic 0.4.11 -> spin 0.10.0 +rockchip-npu 0.2.0 -> spin 0.10.0 +rockchip-soc 0.2.0 -> spin 0.10.0 +scope-local 0.3.7 -> spin 0.10.0 +sg2002-tpu 0.1.1 -> spin 0.10.0 +some-serial 0.4.1 -> spin 0.10.0 +someboot 0.1.15 -> spin 0.10.0 +somehal 0.6.7 -> spin 0.10.0 +starry-kernel 0.5.11 -> spin 0.10.0 +usb-if 0.7.1 -> spin 0.10.0 +x86_vcpu 0.5.8 -> spin 0.10.0 +``` + +Notable root closures from `Cargo.lock`: + +```text +starryos 0.5.11 -> spin 0.10.0, spin 0.9.8 +starry-kernel 0.5.11 -> spin 0.10.0, spin 0.9.8 +ax-std 0.5.14 -> spin 0.10.0, spin 0.9.8 +ax-fs-ng 0.5.14 -> spin 0.10.0, spin 0.9.8 +axfs-ng-vfs 0.4.1 -> spin 0.10.0 +ax-kspin 0.3.8 -> no spin in Cargo.lock closure +``` + +The full reverse closure from `spin 0.9.8` and `spin 0.10.0` contains 93 +packages in the current lockfile. That number includes workspace packages, +test/demo packages, and third-party packages. + +## Direct `Cargo.toml` declarations + +Workspace manifests that declare external `spin` directly: + +```text +components/arm_vcpu/Cargo.toml:23: spin = "0.10" +components/arm_vgic/Cargo.toml:34: spin = "0.10" +components/axaddrspace/Cargo.toml:41: spin = "0.10" +components/axbacktrace/Cargo.toml:20: spin = { version = "0.10", default-features = false, features = ["once"] } +components/axdevice/Cargo.toml:20: spin = "0.10" +components/axdriver_crates/axdriver_net/Cargo.toml:29: spin = "0.9" +components/axfs-ng-vfs/Cargo.toml:20: spin = { version = "0.10", default-features = false, features = ["mutex"] } +components/axfs_crates/axfs_devfs/Cargo.toml:14: spin = "0.9" +components/axfs_crates/axfs_ramfs/Cargo.toml:14: spin = "0.9" +components/axplat_crates/platforms/axplat-aarch64-peripherals/Cargo.toml:18: spin = "0.10" +components/axpoll/Cargo.toml:22: spin = { version = "0.10", default-features = false, features = ["lazy", ...] } +components/axvm/Cargo.toml:21: spin = "0.10" +components/loongarch_vcpu/Cargo.toml:17: spin = "0.10" +components/percpu/percpu/Cargo.toml:41: spin = "0.10" +components/riscv_vplic/Cargo.toml:24: spin = "0.10" +components/scope-local/Cargo.toml:13: spin = { version = "0.10", default-features = false, features = ["lazy"] } +components/someboot/Cargo.toml:43: spin = "0.10" +components/x86_vcpu/Cargo.toml:39: spin = { version = "0.10", default-features = false } +drivers/blk/nvme-driver/Cargo.toml:18: spin = "0.10" +drivers/blk/ramdisk/Cargo.toml:15: spin = "0.10" +drivers/firmware/arm-scmi-rs/Cargo.toml:20: spin = "0.10" +drivers/interface/rdif-serial/Cargo.toml:16: spin = "0.10" +drivers/npu/rockchip-npu/Cargo.toml:21: spin = "0.10" +drivers/rdrive/Cargo.toml:18: spin = "0.10" +drivers/soc/rockchip/rockchip-soc/Cargo.toml:26: spin = "0.10" +drivers/tpu/sg2002-tpu/Cargo.toml:17: spin = "0.10" +drivers/usb/usb-host/Cargo.toml:33: spin = { version = "0.10" } +drivers/usb/usb-if/Cargo.toml:15: spin = "0.10" +os/StarryOS/kernel/Cargo.toml:117: spin = "0.10" +os/arceos/modules/axfs-ng/Cargo.toml:36: spin = { workspace = true } +os/arceos/modules/axfs/Cargo.toml:27: spin = { workspace = true } +os/arceos/modules/axnet-ng/Cargo.toml:34: spin = { workspace = true } +os/arceos/modules/axtask/Cargo.toml:70: spin = { workspace = true, optional = true } +os/axvisor/Cargo.toml:57: spin = "0.10" +platform/axplat-dyn/Cargo.toml:84: spin = "0.10" +platform/somehal/Cargo.toml:31: spin = "0.10" +``` + +The workspace root also defines: + +```text +Cargo.toml:466: spin = "0.10" +``` + +## Source-level direct uses + +Direct Rust source references were counted by primitive: + +```text +Mutex direct lines: 54 +RwLock direct lines: 22 +Once direct lines: 20 +Lazy direct lines: 8 +``` + +Grouped by area: + +```text +17 os/arceos/modules +14 os/StarryOS/kernel + 8 platform/axplat-dyn/src + 8 drivers/usb/usb-host + 4 drivers/rdrive/src + 4 components/arm_vgic/src + 3 os/axvisor/src + 3 os/arceos/api + 3 components/axfs_crates/axfs_ramfs + 2 drivers/tpu/sg2002-tpu + 2 drivers/firmware/arm-scmi-rs + 2 components/axfs_crates/axfs_devfs + 1 platform/somehal/src + 1 drivers/soc/rockchip + 1 drivers/serial/some-serial + 1 drivers/net/realtek-rtl8125 + 1 drivers/interface/rdif-serial + 1 drivers/blk/ramdisk + 1 drivers/blk/nvme-driver + 1 components/x86_vcpu/src + 1 components/scope-local/src + 1 components/riscv_vplic/src + 1 components/percpu/percpu + 1 components/loongarch_vcpu/src + 1 components/kspin/src + 1 components/dma-api/src + 1 components/axvm/src + 1 components/axpoll/src + 1 components/axfs-ng-vfs/src + 1 components/axdriver_crates/axdriver_net + 1 components/axdevice/src + 1 components/axbacktrace/src + 1 components/axaddrspace/tests +``` + +The `components/kspin/src/base.rs` entry is documentation text referencing +`spin::Mutex`, not an actual external dependency from `ax-kspin`. + +## Lockdep-relevant source uses + +These are direct external `spin::{Mutex,RwLock}` uses in kernel/runtime code +that are more likely to matter for lockdep visibility: + +```text +components/arm_vgic/src/v3/gits.rs +components/arm_vgic/src/v3/vgicd.rs +components/arm_vgic/src/v3/vgicr.rs +components/arm_vgic/src/vgic.rs +components/axdevice/src/device.rs +components/axdriver_crates/axdriver_net/src/net_buf.rs +components/axfs-ng-vfs/src/lib.rs +components/axfs_crates/axfs_devfs/src/dir.rs +components/axfs_crates/axfs_ramfs/src/dir.rs +components/axfs_crates/axfs_ramfs/src/file.rs +components/axvm/src/vm.rs +components/dma-api/src/pool.rs +components/loongarch_vcpu/src/registers.rs +components/riscv_vplic/src/vplic.rs +drivers/blk/nvme-driver/src/block.rs +drivers/blk/ramdisk/src/lib.rs +drivers/firmware/arm-scmi-rs/src/lib.rs +drivers/firmware/arm-scmi-rs/src/protocol/mod.rs +drivers/interface/rdif-serial/src/serial.rs +drivers/net/realtek-rtl8125/src/lib.rs +drivers/rdrive/src/lib.rs +drivers/rdrive/src/osal.rs +drivers/rdrive/src/probe/fdt/mod.rs +drivers/rdrive/src/probe/pci/mod.rs +drivers/tpu/sg2002-tpu/src/ion/buffer.rs +drivers/tpu/sg2002-tpu/src/tpu/device.rs +drivers/usb/usb-host/src/backend/kmod/xhci/cmd.rs +drivers/usb/usb-host/src/backend/kmod/xhci/device.rs +drivers/usb/usb-host/src/backend/kmod/xhci/endpoint.rs +drivers/usb/usb-host/src/backend/kmod/xhci/host.rs +drivers/usb/usb-host/src/backend/kmod/xhci/port.rs +drivers/usb/usb-host/src/backend/kmod/xhci/reg.rs +drivers/usb/usb-host/src/backend/kmod/xhci/sync.rs +os/StarryOS/kernel/src/file/mod.rs +os/StarryOS/kernel/src/file/netlink.rs +os/StarryOS/kernel/src/file/signalfd.rs +os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs +os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs +os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs +os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs +os/StarryOS/kernel/src/syscall/fs/lock.rs +os/StarryOS/kernel/src/task/mod.rs +os/StarryOS/kernel/src/task/ops.rs +os/StarryOS/kernel/src/task/posix_timer.rs +os/StarryOS/kernel/src/task/timer.rs +os/arceos/api/arceos_posix_api/src/imp/fd_ops.rs +os/arceos/api/arceos_posix_api/src/imp/pthread/mod.rs +os/arceos/modules/axfs-ng/src/highlevel/file.rs +os/arceos/modules/axfs-ng/src/highlevel/fs.rs +os/arceos/modules/axfs/src/dev.rs +os/arceos/modules/axfs/src/fs/ext4fs.rs +os/arceos/modules/axfs/src/fs/fatfs.rs +os/arceos/modules/axfs/src/root.rs +os/arceos/modules/axnet-ng/src/raw.rs +os/arceos/modules/axnet-ng/src/udp.rs +os/arceos/modules/axnet-ng/src/unix/dgram.rs +os/arceos/modules/axnet/src/smoltcp_impl/udp.rs +os/axvisor/src/hal/arch/loongarch64/mod.rs +os/axvisor/src/vmm/fdt/mod.rs +os/axvisor/src/vmm/vm_list.rs +platform/axplat-dyn/src/drivers/blk/mod.rs +platform/axplat-dyn/src/drivers/blk/virtio_pci.rs +platform/axplat-dyn/src/drivers/mod.rs +platform/axplat-dyn/src/drivers/net/virtio_pci.rs +platform/axplat-dyn/src/drivers/pci.rs +platform/axplat-dyn/src/drivers/soc/scmi.rs +``` + +## Initialization-only source uses + +These direct references are `spin::Once`, `spin::once::Once`, or `spin::Lazy`. +They are usually not lockdep targets because they do not represent ordinary +runtime lock-order edges, although some may still be candidates for replacing +with a project-local initialization primitive later. + +```text +components/axbacktrace/src/lib.rs +components/axfs_crates/axfs_devfs/src/lib.rs +components/axfs_crates/axfs_ramfs/src/lib.rs +components/axpoll/src/lib.rs +components/percpu/percpu/src/imp.rs +components/scope-local/src/scope.rs +drivers/rdrive/src/lib.rs +drivers/rdrive/src/probe/fdt/mod.rs +drivers/rdrive/src/probe/pci/mod.rs +os/StarryOS/kernel/src/pseudofs/dev/ion/mod.rs +os/StarryOS/kernel/src/pseudofs/dev/mod.rs +os/arceos/api/arceos_posix_api/src/imp/stdio.rs +os/arceos/modules/axfs-ng/src/highlevel/fs.rs +os/arceos/modules/axhal/src/dtb.rs +os/arceos/modules/axhal/src/lib.rs +os/arceos/modules/axhal/src/mem.rs +os/arceos/modules/axnet-ng/src/lib.rs +os/arceos/modules/axnet-ng/src/tcp.rs +os/arceos/modules/axtask/src/api.rs +platform/axplat-dyn/src/drivers/blk/rockchip_mmc.rs +platform/axplat-dyn/src/mem.rs +platform/somehal/src/arch/aarch64/systick.rs +``` + +## Test-only source uses + +These direct uses are in test utilities or tests: + +```text +components/axaddrspace/tests/test_utils/mod.rs +components/x86_vcpu/src/test_utils.rs +drivers/serial/some-serial/tests/test.rs +drivers/soc/rockchip/rockchip-soc/tests/test.rs +``` + +## Migration priority + +Suggested priority for eliminating lockdep-relevant blind spots: + +1. `components/axfs-ng-vfs`: directly blocks FAT32/VFS lock-order visibility. + This is the lockdep follow-up recorded in + `os/StarryOS/kernel/src/pseudofs/lockdep-tmpfs-analysis.md`. +2. `os/arceos/modules/axfs-ng`: adjacent to the VFS/FAT/ext4 paths and already + mixes `ax_kspin` with external `spin`. +3. `os/StarryOS/kernel`: Starry runtime locks are user-visible and participate + in lockdep-enabled Starry debug runs. +4. `os/arceos/modules/axnet-ng`, `ax-posix-api`, and `axvisor`: runtime locks + that may matter for broader lockdep coverage. +5. Drivers and portable component crates: migrate only after checking whether + `ax_kspin` is an acceptable dependency boundary for each crate. Some driver + crates may need a smaller synchronization abstraction instead of a direct + ArceOS-specific dependency. +6. `Once`/`Lazy` users: handle separately from `Mutex`/`RwLock`. They are not + the main lockdep visibility gap. + +## Notes + +- Treat external `spin::Mutex` as a busy-wait mutual-exclusion lock, not as a + sleepable mutex. The misleading name should not push migrations toward + `ax_sync::Mutex`; the first replacement target is normally the `ax-kspin` + family, with any later move to a sleepable lock handled as a separate design + change. +- Do not mechanically replace `spin::Mutex` with `ax_kspin::SpinNoPreempt`. + Each site needs a context check: task context, IRQ context, preemption + requirements, and whether the crate is intended to stay OS-neutral. +- Prefer `SpinNoIrq` for replacements that may be acquired from IRQ-enabled + contexts unless the code can prove that the lock is never shared with IRQ + handlers; `SpinNoPreempt` is only safe under that stricter condition. +- `SpinNoIrq` is not a universal repair for `SpinNoPreempt`: if a critical + section can sleep, reschedule, fault on user memory, or call filesystem/device + backends that can do so, the fix is to shorten the critical section or use a + sleepable lock design. +- Current `SpinNoPreempt` follow-ups: + - `components/axfs-ng-vfs`: the migration corrected the lock flavor to + `SpinNoIrq`; backend callbacks under VFS spin locks are now intentionally + left as a separately exposed follow-up issue. + - `os/arceos/modules/axfs-ng` FAT/ext4: large filesystem locks around I/O and + flush paths need a broader lock strategy, not a mechanical IRQ-disabling + replacement. + - Starry `epoll`, `pty`, and terminal metadata: short critical sections that + can be considered for `SpinNoIrq` after wakeup/tty ordering review. + - Starry loop-device cache: tied to the ext4 block-device path and should be + reviewed with the axfs-ng ext4 lock strategy. +- If migrating `axfs-ng-vfs` exposes the suspected FAT32/VFS ABBA ordering, the + fix should be a real ordering fix, not a lockdep subclass annotation. diff --git a/reports/external-spin-migration-plan.md b/reports/external-spin-migration-plan.md new file mode 100644 index 0000000000..8e4ae0874a --- /dev/null +++ b/reports/external-spin-migration-plan.md @@ -0,0 +1,291 @@ +# External `spin` migration plan + +Date: 2026-05-21 + +This plan follows the audit in +[`reports/external-spin-audit.md`](external-spin-audit.md). The goal is not only +to remove a third-party dependency from the build graph, but also to make the +lock migration process reviewable for future maintainers. + +## Background + +The workspace currently uses the external `spin` crate in two ways: + +- direct `spin::{Mutex,RwLock,Once,Lazy}` use from TGOSKits crates; +- indirect use through third-party crates, most notably `lazy_static` with the + `spin_no_std` feature. + +The original lockdep follow-up was triggered by a visibility gap: external +`spin::{Mutex,RwLock}` locks do not participate in `ax-kspin` lockdep tracking. +However, replacing all `spin` use with `ax-kspin` in one step is not realistic, +because `ax-kspin` currently provides mutex-like spin locks only. It does not +provide `RwLock`, `Once`, or `Lazy`. + +Therefore the migration should be split into two separate tracks: + +- supply-chain decoupling: bring the external `spin` implementation into the + repository so builds no longer depend on fetching it from crates.io or a + registry mirror; +- semantic migration: gradually replace lockdep-relevant `spin` locks with + TGOSKits-native synchronization primitives. + +## Decision + +Keep a local copy of the newer external `spin` implementation as an internal +component, initially based on `spin 0.10.0`. + +The local component should remain API-compatible with upstream `spin` at first. +This keeps the first change low risk: existing `spin::Mutex`, `spin::RwLock`, +`spin::Once`, and `spin::Lazy` users continue to compile while the dependency is +resolved from the repository. + +The upstream name `spin::Mutex` is semantically misleading in this kernel +context. It is a busy-wait mutual-exclusion primitive and has no path to sleep +while waiting. Therefore existing `spin::Mutex` users should first be treated as +non-sleeping locks and migrated into the `ax-kspin` family. Replacing them with +`ax_sync::Mutex` is a separate semantic change and should only happen after a +site proves that a sleepable lock is the correct design. + +This first stage does not make those locks visible to lockdep. It only makes the +codebase independent from the external crate source and gives the project a +controlled place for future compatibility and migration work. + +## Phase 1: remove workspace `spin 0.9` users + +The lockfile currently contains both `spin 0.9.8` and `spin 0.10.0`. + +Workspace-owned `spin = "0.9"` declarations should be upgraded first: + +```text +components/axfs_crates/axfs_ramfs/Cargo.toml +components/axfs_crates/axfs_devfs/Cargo.toml +components/axdriver_crates/axdriver_net/Cargo.toml +``` + +These crates use only APIs that are still present in `spin 0.10`: + +```text +spin::Mutex +spin::RwLock +spin::once::Once +``` + +After this step, any remaining `spin 0.9.8` should come from third-party +dependencies rather than direct workspace declarations. + +## Phase 2: vendor `spin 0.10` + +Add a local component for `spin 0.10.0`, preserving license and upstream +attribution. + +Expected location: + +```text +components/spin +``` + +The package should still be named `spin`, so existing source imports do not need +to change in this phase. + +Cargo resolution should then be redirected to the local component, for example +with a root-level patch: + +```toml +[patch.crates-io] +spin = { path = "components/spin" } +``` + +This should remove external registry dependency for `spin 0.10.0` while keeping +behavior unchanged. + +## Phase 3: analyze and contain `lazy_static` + +`lazy_static 1.5.0` with `spin_no_std` hard-codes: + +```toml +spin = { version = "0.9.8", features = ["once"], default-features = false } +``` + +Its no-std implementation uses only `spin::Once`. This is not a primary lockdep +blind spot because it is an initialization primitive rather than a normal +runtime `Mutex` or `RwLock`. + +For that reason, `lazy_static -> spin 0.9.8` should be treated as a separate +follow-up, not as a blocker for vendoring `spin 0.10`. + +Possible resolutions: + +- replace workspace `lazy_static!` call sites with `spin::Lazy`, + `ax_lazyinit::LazyInit`, or a future TGOSKits-native `Once/Lazy`; +- vendor or patch `lazy_static` so its no-std path uses the local `spin 0.10` or + a project-local Once primitive; +- keep it temporarily and document the remaining `spin 0.9.8` lockfile entry as + an initialization-only residual dependency. + +## Phase 4: semantic migration to TGOSKits primitives + +Once `spin` is local and controlled, replace lockdep-relevant uses in priority +order. + +Priority: + +1. `components/axfs-ng-vfs` + - This directly addresses the FAT32/VFS lockdep visibility gap. +2. `os/arceos/modules/axfs-ng` + - Adjacent to VFS/FAT/ext4 paths and already mixes `ax_kspin` with external + `spin`. +3. `os/StarryOS/kernel` + - User-visible runtime locks that matter for Starry lockdep/debug runs. +4. `os/arceos/modules/axnet-ng`, `os/arceos/api/arceos_posix_api`, and + `os/axvisor` + - Runtime locks that may matter for broader lockdep coverage. +5. Drivers and portable components + - Migrate only after checking dependency boundaries. Some driver crates may + need a small synchronization abstraction instead of directly depending on + ArceOS-specific `ax-kspin`. + +Replacement rules: + +- `spin::Mutex` can usually become one of: + - `ax_kspin::SpinNoPreempt`; + - `ax_kspin::SpinNoIrq`; + - `ax_kspin::SpinRaw`. +- Do not read the upstream `Mutex` name as equivalent to `ax_sync::Mutex`. + `spin::Mutex` is non-sleeping and busy-waits, so the migration default is an + `ax-kspin` primitive. Moving a site to `ax_sync::Mutex` is a later design + decision, not a mechanical replacement. +- `spin::RwLock` is not directly covered by `ax-kspin` today. + - Some sites may be safely downgraded to a mutex. + - Read-heavy shared structures need a real internal RwLock design or a + separate migration decision. +- `spin::Once` and `spin::Lazy` should be handled separately from lockdep + mutex/RwLock migration. + +Do not mechanically replace every `spin::Mutex` with `SpinNoPreempt`. Each site +needs a context check: + +- whether it can run in IRQ context; +- whether lock acquisition itself happens with local IRQs enabled; +- whether the critical section can sleep, reschedule, fault on user memory, or + call into a backend callback that can do so; +- whether preemption must be disabled; +- whether it is already protected by an outer critical section; +- whether the crate is meant to stay OS-neutral; +- whether lockdep visibility is actually required. + +Follow-up `SpinNoPreempt` audit after the first VFS/axfs-ng migrations: + +- `components/axfs-ng-vfs` initially aliased its internal VFS locks to + `SpinNoPreempt`. That exposed a real Starry tmpfs panic when + `Location::mount()` held a VFS mountpoint lock and called the filesystem + backend's `root_dir()`. The immediate migration correction is to keep VFS + metadata locks in the `ax-kspin` family but use `SpinNoIrq`. The backend + callback issue should remain visible to `might_sleep`/lockdep and be handled + as a separate lock-scope follow-up, not as part of the spin replacement step. +- `os/arceos/modules/axfs-ng` FAT and ext4 filesystem locks also use + `SpinNoPreempt`. They protect large filesystem states and often wrap block + I/O, sync, and flush paths. They are not good candidates for a mechanical + `SpinNoIrq` replacement; they may need sleepable, lockdep-visible locking or + smaller critical sections. +- Starry `epoll`, `pty`, and terminal metadata use short `SpinNoPreempt` + critical sections. They are likely candidates for `SpinNoIrq` if the call + sites are IRQ-enabled, but should still be reviewed for wakeup and tty lock + ordering. +- Starry loop-device cache locking is tied to the ext4 block-device path and + should be considered together with the axfs-ng ext4 lock strategy. + +## Phase 4 status: production `spin::Mutex` migration + +As of commit `44af7d3a1`, direct production uses of `spin::Mutex` and +`spin::MutexGuard` have been migrated away from workspace code, excluding the +vendored `components/spin` implementation itself. + +The verification command: + +```text +rg -n "^use spin::Mutex|^use spin::\{[^}]*Mutex|spin::Mutex|spin::MutexGuard|spin::mutex::" \ + --glob '*.rs' --glob '!components/spin/**' +``` + +now reports only: + +```text +components/kspin/src/base.rs +``` + +This is a documentation reference to the original upstream implementation, not +a runtime lock site. + +The production migration covered these groups: + +- filesystem and VFS paths: `axfs-ng-vfs`, `axfs-ng`, `ax-fs`; +- networking and Starry runtime paths: `ax-net-ng`, Starry timer/netlink/usbfs + and camera locks; +- virtualization and platform components: `axvisor`, `axvm`, `riscv_vplic`, + `loongarch_vcpu`, `arm_vgic`, `axplat-dyn`; +- portable driver components: `rdrive`, `arm-scmi-rs`, `ramdisk`, + `nvme-driver`, `rdif-serial`, `realtek-rtl8125`, `sg2002-tpu`, `crab-usb`; +- shared support crates: `dma-api`, `ax-driver-net`, `axdevice`. + +The repository still keeps local `spin` for intentionally separate work: + +- `spin::Once` and `spin::Lazy` initialization primitives; +- postponed `spin::RwLock` users; +- documentation references. + +To make the completed `spin::Mutex` migration visible to Cargo, the local +`components/spin` default feature set no longer enables `mutex`, `spin_mutex`, +or `barrier`. The mutex implementation remains in the vendored component behind +explicit opt-in features, but default workspace users should not be able to name +`spin::Mutex` by accident. Any remaining default-feature consumer that still +uses `spin::Mutex`, `spin::MutexGuard`, or `spin::mutex::*` should now fail at +compile time. + +Any lock-scope bugs exposed by `might_sleep` or lockdep after this migration +should be treated as useful follow-up findings. They are not a reason to hide +the original non-sleeping `spin::Mutex` semantics behind a sleepable +`ax_sync::Mutex` replacement. + +## Validation strategy + +For documentation-only planning changes, no build is required. + +For future implementation changes: + +- run `cargo fmt`; +- for each modified crate, run targeted clippy, preferably: + +```text +cargo xtask clippy --package +``` + +- for `ax-kspin` or lockdep changes, also run the relevant lockdep tests when + practical; +- for VFS/FAT changes, add or run a targeted FAT32/VFS case after the lock type + migration, because improved lockdep visibility may expose the suspected ABBA + ordering. + +If `cargo tree` is needed, prefer using it only after dependency resolution is +known to work locally. During the audit, normal and offline `cargo tree` were +blocked by registry/index state around `sg200x-bsp = 0.6.0`, while direct +`Cargo.lock` parsing remained reliable. + +## Expected milestones + +1. Direct workspace `spin 0.9` declarations are upgraded to `spin 0.10`. +2. `components/spin` is added and `spin 0.10` resolves locally. +3. Remaining `spin 0.9` entries, if any, are attributed to `lazy_static` or other + third-party dependencies. +4. `components/axfs-ng-vfs` no longer uses external `spin::{Mutex,RwLock}` for + lockdep-relevant internal locks. +5. `os/arceos/modules/axfs-ng` moves lockdep-relevant runtime locks away from + external `spin`; `FS_REGISTRY` is the first small step, while + `CachedFile::append_lock` remains a separate `RwLock` design question. +6. Lockdep-enabled FAT32/VFS testing either confirms no report or exposes a real + ordering issue for a separate ordering fix. +7. Production direct `spin::Mutex` and `spin::MutexGuard` uses are gone outside + the vendored `components/spin` crate. The remaining direct match is limited + to an `ax-kspin` documentation reference. +8. The vendored `components/spin` default features no longer expose `Mutex`. + This turns accidental new default-feature `spin::Mutex` users into compile + errors while preserving explicit compatibility features for the local copy. diff --git a/reports/spin-no-preempt-audit.md b/reports/spin-no-preempt-audit.md new file mode 100644 index 0000000000..5324551b36 --- /dev/null +++ b/reports/spin-no-preempt-audit.md @@ -0,0 +1,354 @@ +# `SpinNoPreempt` usage audit + +Date: 2026-05-22 + +This note records the current `SpinNoPreempt` and direct `NoPreempt` usage +sites. It is a follow-up to the external `spin` migration notes and focuses on +the risks introduced by disabling preemption without disabling local IRQs. + +## Rule of thumb + +`ax_kspin::SpinNoPreempt` is an atomic-context spin lock: + +- locking disables kernel preemption; +- locking does not disable local IRQs; +- lockdep tracks it when `ax-kspin/lockdep` is enabled. + +The API documentation in `components/kspin/src/lib.rs` says it must either be +used while local IRQs are already disabled, or never be used from interrupt +handlers. + +That creates two independent hazards: + +1. Same-CPU IRQ reentry deadlock. If a task holds a `SpinNoPreempt` lock with + local IRQs enabled and an IRQ handler or IRQ-triggered waker tries to acquire + the same lock on the same CPU, the handler spins forever because the lock + holder cannot resume. +2. Atomic-context sleep violation. While the guard is alive, + `axtask::might_sleep()` sees `preempt_count != 0`. User-memory page faults, + blocking mutexes, scheduler paths, and filesystem or device callbacks that + can sleep must not run under the guard. + +`SpinNoIrq` only addresses the first hazard. It still creates atomic context and +is not a repair for code that may sleep, reschedule, fault on user memory, or +call a backend that may do so. + +## Follow-up principle: do not treat spin locks as the final lock model + +Several current `SpinNoIrq` changes are compatibility fixes for today's Starry +boot and test paths, not a statement that the protected state should +permanently use spin locks. Some of the observed failures came from +`might_sleep()` checks in very early boot or rootfs setup. Linux's equivalent +debug checks are effectively stage-aware: before normal scheduling and +concurrency are established, some early-boot paths do not have the same +sleepability constraints as runtime task context. This project currently lacks +that nuance, so a `might_sleep()` panic can indicate either a real runtime bug +or an overly strict early-boot classification. + +Future fixes should prefer restoring sleepable `Mutex` usage where the data +structure protects filesystem, VFS, block I/O, allocation-heavy, or user-memory +paths. The better long-term directions are: + +- make `might_sleep()` or its callers aware of early boot and scheduler state; +- move rootfs mount, filesystem flush, and similar work into a normal sleepable + task context when possible; +- split coarse locks so they protect only in-memory metadata mutation and are + dropped before block I/O, VFS callbacks, user-copy, or page-faultable access; +- keep `SpinNoIrq` only for short, bounded, non-sleeping critical sections that + genuinely need IRQ-safe exclusion. + +In short, `Mutex -> SpinNoIrq` should remain a conservative stopgap for the +current failure mode. It should not become the default response to +`might_sleep()` reports. + +## Search commands + +```text +rg -n "SpinNoPreempt|SpinNoPreemptGuard|BaseSpinLock>` uses the same `SpinNoIrq` alias after the +change. It is used only to install and clone the root dentry and can later +become `OnceCell` if desired. + +### ext4 with `rsext4` + +`os/arceos/modules/axfs-ng/src/fs/ext4/rsext4/fs.rs` used to alias +`SpinNoPreempt` as the filesystem mutex protecting `Ext4State`. It now uses +`SpinNoIrq`. + +Risk: + +- `sync_to_disk()` holds the lock while flushing all block, bitmap, and inode + caches, syncing the superblock and group descriptors, committing the journal, + and calling `dev.cantflush()`. +- `read_at()` holds the lock while resolving extents and loading data blocks. +- `write_at()`, `append()`, `set_len()`, `set_symlink()`, `create()`, `link()`, + `unlink()`, and `rename()` hold the lock across rsext4 operations that may + allocate, touch caches, and call the block device. +- The loop-device adapter keeps `flush()` as a no-op to avoid re-entering + backing-file VFS writeback from filesystem block I/O paths. + +Assessment: the `SpinNoPreempt` same-CPU IRQ reentry hazard is resolved by +moving the filesystem mutex to `SpinNoIrq`. This is a conservative compatibility +fix for current Starry boot/rootfs paths; it does not make the coarse lock safe +for sleeping work. Disk, cache, and journal operations can still run in atomic +context while the guard is held, so the longer-term fix is to avoid invoking +these filesystem paths before the kernel has a sleepable task context, or to +split the filesystem serialization strategy. + +### ext4 with `lwext4` + +`os/arceos/modules/axfs-ng/src/fs/ext4/lwext4/fs.rs` used the same +`SpinNoPreempt` alias for `LwExt4Filesystem`. It now uses `SpinNoIrq`. + +Risk: + +- `read_at`, `write_at`, `append`, `set_len`, `set_symlink`, `read_dir`, + `lookup`, `create`, `link`, `unlink`, and `rename` call into `lwext4_rust` + while holding the lock. +- The `flush()` implementation directly calls `self.inner.lock().flush()`. + +Assessment: the `SpinNoPreempt` same-CPU IRQ reentry hazard is resolved by +moving the filesystem mutex to `SpinNoIrq`. As with rsext4, this keeps current +early/rootfs contexts working but leaves the broader coarse-lock and block-I/O +under atomic context issue for a later design change. + +## Starry epoll + +`os/StarryOS/kernel/src/file/epoll.rs` used to use `SpinNoPreempt` for: + +- `EpollInterest::mode`; +- `EpollInner::interests`; +- `EpollInner::ready_queue`. + +All three now use `SpinNoIrq`. `ready_queue` needed the change because it can +be touched by `InterestWaker::wake_by_ref()`, and wakers may be invoked from IRQ +wake paths. `mode` and `interests` are short critical sections without a proven +outer IRQ-disabled context, so they also follow the conservative rule that +`SpinNoPreempt` should not be used there. + +Risk: + +- `mode` is a short state lock. It is currently taken in task-side epoll paths + and does not wrap user-memory access or blocking operations. +- `interests` is taken in `add`, `modify`, `delete`, and stale-entry removal. + It wraps `HashMap` mutation and some `Arc` replacement/drop work, but does not + call `FileLike::poll` or `register` while held. +- `ready_queue` is different from `mode` and `interests`: + `InterestWaker::wake_by_ref()` pushes into it. That waker can be invoked by a + `PollSet::wake()` path. Some poll sets are woken from IRQ handlers, for + example the Starry UART IRQ path calls `poll.wake()` after filling its RX + buffer. + +Assessment: medium residual risk. Moving the epoll locks to `SpinNoIrq` closes +the immediate same-CPU IRQ reentry hole. It does not solve the fact that +`VecDeque::push_back` may allocate from a waker path. A follow-up should make +epoll wake enqueueing IRQ-safe explicitly by preallocating/bounding the queue or +deferring heap-growing work out of IRQ context. `interests` also uses a +`HashMap`, but that path is driven by `epoll_ctl` style task-context operations, +not by the waker fast path. + +## Starry loop-device cache + +`os/StarryOS/kernel/src/pseudofs/dev/loop.rs` used +`SpinNoPreempt>>` for `CacheData::blocks`. It now uses +`SpinNoIrq`, because ext4 can call the loop block-device adapter while holding +filesystem locks that already put the kernel in atomic context. A blocking +mutex panicked in the `util-linux` Starry QEMU case when `read_block()` tried to +lock the cache under the ext4 filesystem guard. + +Risk: + +- `LoopBlockDevice::{read_block,write_block}` hold the lock only while copying + data between the cache and caller buffers and updating the dirty flag. +- `writeback_buffer()` copies one cache chunk into a stack buffer while holding + the lock, then drops the lock before calling `FileBackend::write_at` or + `sync`. +- The comments explicitly avoid doing VFS writeback from filesystem block I/O + paths. + +Assessment: the `SpinNoPreempt` same-CPU IRQ reentry hazard is resolved by +using `SpinNoIrq`, and the observed blocking-mutex panic is avoided without +re-entering VFS writeback from ext4 block-I/O callbacks. The critical sections +remain bounded memory copies. If this cache ever needs to allocate or perform +VFS I/O under the guard, the strategy must be revisited. + +## Starry tmpfs and VFS cache nesting + +`os/StarryOS/kernel/src/pseudofs/tmp.rs` now uses `SpinNoIrq` for the tmpfs +root dentry and per-directory entry maps. + +Risk: + +- `MemoryFs::root_dir()` is called while mounting pseudofs during startup. + Using a blocking mutex there caused `might_sleep()` to panic when the startup + path reached tmpfs before a sleepable context was available. +- `components/axfs-ng-vfs/src/node/dir.rs` protects its dentry cache with + `SpinNoIrq` and calls filesystem `lookup`, `create`, `unlink`, and + `open_file` paths while the cache guard is held. A blocking tmpfs directory + map mutex therefore panicked when a tmpfs cache miss reached + `MemoryNode::lookup()`. +- `MemoryNode::link()` no longer calls the generic `DirEntry::metadata()` while + holding the tmpfs entries guard; it reads the target tmpfs inode metadata + directly from the short `SpinNoIrq` metadata lock. + +Assessment: the observed Starry QEMU panics are resolved by keeping tmpfs root +and directory maps non-blocking. This remains a compatibility fix for the +current VFS cache contract. A broader VFS improvement would avoid calling +filesystem operations while holding the dentry-cache spin guard, which would +let tmpfs directory maps move back to a sleepable lock if desired. + +## Unix domain socket path binding + +`os/arceos/modules/axnet-ng/src/unix/mod.rs` no longer executes transport +callbacks while holding `DirEntry::user_data()`'s `SpinNoIrq` guard. + +Risk: + +- `/dev/log` setup creates a Unix datagram socket and binds it to a filesystem + path during Starry pseudofs initialization. +- Path-based Unix sockets store their `BindSlot` in VFS `user_data`, whose lock + is a `SpinNoIrq` guard from `axfs-ng-vfs`. +- The old code passed a borrowed `BindSlot` directly into the callback while + the `user_data` guard was still alive. `DgramTransport::bind()` then tried to + take its sleepable socket mutex under that spin guard and tripped + `might_sleep()`. + +Assessment: resolved by cloning the `Arc` out of `user_data`, dropping +the spin guard, and only then invoking the transport operation. This keeps +socket internals sleepable while preserving the short VFS user-data critical +section. + +## Starry packet and netlink receive paths + +`os/StarryOS/kernel/src/file/netlink.rs` and +`os/StarryOS/kernel/src/file/packet.rs` hit the second `SpinNoIrq` hazard: +faultable user-memory writes while a spin guard was alive. + +Risk: + +- `NetlinkSocket::read_one()` used to pop and copy the queued netlink message + while holding the socket queue guard. `c-regression/test-netlink-genl` + faulted during `IoDst::write()` and tripped the + "faultable user memory access requires IRQs enabled" assertion. +- `PacketSocket::recv_packet()` used to write the pending packet into the user + buffer inside the socket state critical section. `bugfix/bug-packet-arping` + reproduced the same user-copy assertion. + +Assessment: resolved by moving the queued packet/message into a local variable +under the guard, dropping the guard, and only then copying into user memory. +The locks still protect only queue/state mutation; page-faultable user access +now happens with IRQs enabled. + +## Starry tty metadata + +`os/StarryOS/kernel/src/pseudofs/dev/tty/terminal/mod.rs` used +`SpinNoPreempt` for: + +- `window_size`; +- `termios`. + +Both now use `SpinNoIrq`, because these short locks were taken without a proven +outer IRQ-disabled context. + +Risk: + +- ioctl paths copy user data before acquiring the lock on write-side updates. + Existing comments call out that user-memory access under the guard would page + fault and panic in `might_sleep()`. +- read-side paths clone/copy the data under the lock and then perform later + work after the guard is dropped. +- The lock is not currently taken from the serial IRQ handler; IRQ paths wake a + `PollSet` and do not touch terminal termios/window-size state. + +Assessment: low risk if the current pattern is preserved. Keep the rule that +all `vm_read`, `vm_write`, blocking operations, and line-discipline work happen +outside the guard. A small helper that loads/stores termios/window-size by value +would reduce the chance of future call sites accidentally extending the guard +lifetime. + +## Starry pty producer + +`os/StarryOS/kernel/src/pseudofs/dev/tty/pty.rs` used +`Arc>>` in `PtyWriter`. It now uses `SpinNoIrq`, +because the producer lock is not taken under a proven outer IRQ-disabled +context. + +Risk: + +- `write()` holds the lock only for `push_slice(buf)`. +- `PollSet::wake()` is called after the guard is dropped. +- The copied amount is bounded by the 4 KiB PTY buffer. + +Assessment: low risk. The current lock does not wrap a wakeup, user-memory +access, or blocking operation. If future writers can run directly in IRQ +context, change the lock strategy; with the current task-side writer model this +can remain a short spin critical section. + +## Direct `NoPreempt` users + +`os/arceos/modules/axhal/src/irq.rs` creates a `NoPreempt` guard in +`handle_irq()`. This is intentional: the function already runs in interrupt +context, so local IRQs are expected to be disabled by the trap path, and the +guard prevents scheduler preemption until the handler returns. + +`components/percpu/percpu/src/custom/mod.rs` and generated percpu macro code use +`NoPreempt` only around current-CPU percpu access. That prevents migration while +accessing CPU-local storage and does not protect shared data with a spin lock. +It is outside the main `SpinNoPreempt` lock audit, but the same rule applies: +do not add sleeping work inside those guarded closures. + +## Recommended order + +1. Review the residual `epoll.ready_queue` allocation path after the IRQ-safe + lock changes. +2. Keep tty termios/window-size and pty producer locks as short `SpinNoIrq` + critical sections, but add helper APIs or comments if future edits start + extending guard lifetimes. +3. Leave `axhal` IRQ and percpu `NoPreempt` guards as intentional uses unless a + specific sleeping path is introduced under them. diff --git a/scripts/axbuild/src/build.rs b/scripts/axbuild/src/build.rs index 994c1d3b24..9be469717e 100644 --- a/scripts/axbuild/src/build.rs +++ b/scripts/axbuild/src/build.rs @@ -175,6 +175,7 @@ impl BuildInfo { ) -> anyhow::Result { if self.std_build { self.validated_max_cpu_num()?; + self.resolve_std_features(); let std_target = std_build_target_for(target)?; let mut cargo = self.into_base_cargo_config_with_log( package.to_string(), @@ -203,6 +204,16 @@ impl BuildInfo { Ok(self.into_base_cargo_config_with_log(package.to_string(), target.to_string(), args)) } + fn resolve_std_features(&mut self) { + self.features = self + .features + .iter() + .map(|feature| normalize_std_feature(feature)) + .collect(); + self.features.sort(); + self.features.dedup(); + } + pub(crate) fn prepare_non_dynamic_platform_for( &mut self, package: &str, @@ -587,6 +598,19 @@ fn normalize_legacy_feature_alias(feature: &str) -> String { } } +fn normalize_std_feature(feature: &str) -> String { + let normalized = normalize_legacy_feature_alias(feature); + match normalized.as_str() { + "ax-std" | "ax-feat" => normalized, + feature if feature.starts_with("ax-std/") || feature.starts_with("ax-feat/") => feature + .split_once('/') + .map(|(_, feature)| format!("arceos-rust/{feature}")) + .unwrap_or_else(|| normalized.clone()), + feature if feature.starts_with("arceos-rust/") => normalized, + feature => format!("arceos-rust/{feature}"), + } +} + pub(crate) fn parse_makefile_features(input: &str) -> Vec { let mut features = Vec::new(); for feature in input.split(|ch: char| ch == ',' || ch.is_whitespace()) { @@ -648,6 +672,11 @@ fn apply_makefile_features_with_prefix_family( return; } + if build_info.std_build { + apply_std_makefile_features(build_info, makefile_features); + return; + } + let prefix_family = build_info.resolve_ax_feature_prefix_family(package, prefix_family); for feature in makefile_features { @@ -669,6 +698,19 @@ fn apply_makefile_features_with_prefix_family( } } +fn apply_std_makefile_features(build_info: &mut BuildInfo, makefile_features: &[String]) { + for feature in makefile_features { + let mapped = normalize_std_feature(feature); + if !build_info + .features + .iter() + .any(|existing| existing == &mapped) + { + build_info.features.push(mapped); + } + } +} + pub(crate) fn default_build_info_path_in_workspace( workspace_root: &Path, package: &str, @@ -1147,6 +1189,45 @@ mod tests { assert_eq!(family, AxFeaturePrefixFamily::AxFeat); } + #[test] + fn std_build_maps_arceos_features_to_arceos_rust_dependency() { + let mut info = BuildInfo { + std_build: true, + features: vec![ + "ax-std".to_string(), + "lockdep".to_string(), + "axstd/smp".to_string(), + ], + ..BuildInfo::default() + }; + + info.resolve_std_features(); + + assert!(info.features.contains(&"ax-std".to_string())); + assert!(info.features.contains(&"arceos-rust/lockdep".to_string())); + assert!(info.features.contains(&"arceos-rust/smp".to_string())); + assert!(!info.features.contains(&"ax-std/lockdep".to_string())); + assert!(!info.features.contains(&"lockdep".to_string())); + } + + #[test] + fn makefile_features_use_arceos_rust_prefix_for_std_build() { + let mut info = BuildInfo { + std_build: true, + features: Vec::new(), + ..BuildInfo::default() + }; + + apply_makefile_features_with_prefix_family( + &mut info, + "test-arceos-std-app", + &[String::from("lockdep")], + Err(anyhow::anyhow!("std test packages do not depend on ax-std")), + ); + + assert_eq!(info.features, vec!["arceos-rust/lockdep".to_string()]); + } + #[test] fn resolve_platform_package_prefers_matching_explicit_platform_dependency() { let metadata = repo_metadata(); diff --git a/test-suit/arceos/rust/task/lockdep/Cargo.toml b/test-suit/arceos/rust/task/lockdep/Cargo.toml index dce244aba5..1ed67a4d78 100644 --- a/test-suit/arceos/rust/task/lockdep/Cargo.toml +++ b/test-suit/arceos/rust/task/lockdep/Cargo.toml @@ -7,9 +7,10 @@ description = "A regression test for ArceOS lockdep lock-order detection" publish = false [features] -ax-std = ["dep:ax-std", "dep:ax-kspin"] +ax-std = ["dep:ax-std", "dep:ax-kspin", "dep:axfs-ng-vfs"] lockdep = ["ax-std", "ax-std/lockdep"] [dependencies] ax-std = { workspace = true, features = ["alloc", "multitask"], optional = true } ax-kspin = { workspace = true, optional = true } +axfs-ng-vfs = { workspace = true, optional = true } diff --git a/test-suit/arceos/rust/task/lockdep/README.md b/test-suit/arceos/rust/task/lockdep/README.md index 41c82e00c6..db6364400f 100644 --- a/test-suit/arceos/rust/task/lockdep/README.md +++ b/test-suit/arceos/rust/task/lockdep/README.md @@ -12,6 +12,7 @@ This test app exercises lock order inversion detection for ArceOS lockdep. - `mixed-two-task`: two-task spin->mutex then mutex->spin - `mixed-ms-single`: single-task mutex->spin then spin->mutex - `mixed-ms-two-task`: two-task mutex->spin then spin->mutex +- `vfs-cache-single`: single-task `axfs-ng-vfs` dentry user-data/cache ABBA ## Test modes @@ -138,3 +139,13 @@ FEATURES=lockdep LOCKDEP_CASE=mixed-ms-two-task cargo xtask arceos qemu \ --config test-suit/arceos/rust/task/lockdep/build-x86_64-unknown-none.toml \ --qemu-config test-suit/arceos/rust/task/lockdep/qemu-x86_64.toml ``` + +Run the VFS cache visibility case with lockdep on x86_64: + +```bash +FEATURES=lockdep cargo xtask arceos qemu \ + --package arceos-lockdep \ + --target x86_64-unknown-none \ + --config test-suit/arceos/rust/task/lockdep/build-vfs-cache-x86_64-unknown-none.toml \ + --qemu-config test-suit/arceos/rust/task/lockdep/qemu-x86_64.toml +``` diff --git a/test-suit/arceos/rust/task/lockdep/build-vfs-cache-x86_64-unknown-none.toml b/test-suit/arceos/rust/task/lockdep/build-vfs-cache-x86_64-unknown-none.toml new file mode 100644 index 0000000000..79dfed1ae8 --- /dev/null +++ b/test-suit/arceos/rust/task/lockdep/build-vfs-cache-x86_64-unknown-none.toml @@ -0,0 +1,8 @@ +features = ["ax-std"] +log = "Warn" +max_cpu_num = 4 + +[env] +AX_GW = "10.0.2.2" +AX_IP = "10.0.2.15" +LOCKDEP_CASE = "vfs-cache-single" diff --git a/test-suit/arceos/rust/task/lockdep/src/main.rs b/test-suit/arceos/rust/task/lockdep/src/main.rs index daae6a5a52..b7ffd199c1 100644 --- a/test-suit/arceos/rust/task/lockdep/src/main.rs +++ b/test-suit/arceos/rust/task/lockdep/src/main.rs @@ -19,11 +19,15 @@ app! { #[cfg(feature = "ax-std")] extern crate ax_std as std; +#[cfg(feature = "ax-std")] +use core::any::Any; + #[cfg(feature = "ax-std")] use std::{ + string::ToString, sync::{ Arc, Mutex, - atomic::{AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, }, thread, }; @@ -31,6 +35,13 @@ use std::{ #[cfg(feature = "ax-std")] use ax_kspin::SpinRaw; +#[cfg(feature = "ax-std")] +use axfs_ng_vfs::{ + DeviceId, DirEntry, DirEntrySink, DirNode, DirNodeOps, FilesystemOps, Metadata, + MetadataUpdate, NodeFlags, NodeOps, NodePermission, NodeType, Reference, StatFs, VfsError, + VfsResult, WeakDirEntry, +}; + #[cfg(feature = "ax-std")] const WAIT_UNTIL_RETRY_LIMIT: usize = 10_000_000; @@ -43,13 +54,15 @@ const WAIT_UNTIL_RETRY_LIMIT: usize = 10_000_000; // - mixed-two-task: two-task spin->mutex then mutex->spin // - mixed-ms-single: single-task mutex->spin then spin->mutex // - mixed-ms-two-task: two-task mutex->spin then spin->mutex +// - vfs-cache-single: single-task axfs-ng-vfs dentry cache ABBA #[cfg(feature = "ax-std")] fn lockdep_case() -> &'static str { match option_env!("LOCKDEP_CASE") { Some(case) => case, None => panic!( "LOCKDEP_CASE is required; choose one of: mutex-single, mutex-two-task, spin-single, \ - spin-two-task, mixed-single, mixed-two-task, mixed-ms-single, mixed-ms-two-task" + spin-two-task, mixed-single, mixed-two-task, mixed-ms-single, mixed-ms-two-task, \ + vfs-cache-single" ), } } @@ -247,6 +260,166 @@ fn mixed_ms_two_task_abba() { handle.join().unwrap(); } +#[cfg(feature = "ax-std")] +struct TestFs; + +#[cfg(feature = "ax-std")] +impl FilesystemOps for TestFs { + fn name(&self) -> &str { + "lockdep-vfs-test" + } + + fn root_dir(&self) -> DirEntry { + panic!("not used by lockdep-vfs-test") + } + + fn stat(&self) -> VfsResult { + Err(VfsError::Unsupported) + } +} + +#[cfg(feature = "ax-std")] +static TEST_FS: TestFs = TestFs; + +#[cfg(feature = "ax-std")] +struct TestDir { + inode: u64, + this: WeakDirEntry, + renamed: AtomicBool, +} + +#[cfg(feature = "ax-std")] +impl TestDir { + fn new_child(&self, name: &str) -> DirEntry { + DirEntry::new_dir( + |this| { + DirNode::new(Arc::new(Self { + inode: self.inode + 100, + this, + renamed: AtomicBool::new(false), + })) + }, + Reference::new(self.this.upgrade(), name.to_string()), + ) + } + + fn new_entry(inode: u64, name: &str) -> DirEntry { + DirEntry::new_dir( + |this| { + DirNode::new(Arc::new(Self { + inode, + this, + renamed: AtomicBool::new(false), + })) + }, + Reference::new(None, name.to_string()), + ) + } +} + +#[cfg(feature = "ax-std")] +impl NodeOps for TestDir { + fn inode(&self) -> u64 { + self.inode + } + + fn metadata(&self) -> VfsResult { + Ok(Metadata { + device: 0, + inode: self.inode, + nlink: 1, + mode: NodePermission::default(), + node_type: NodeType::Directory, + uid: 0, + gid: 0, + size: 0, + block_size: 4096, + blocks: 0, + rdev: DeviceId::default(), + atime: Default::default(), + mtime: Default::default(), + ctime: Default::default(), + }) + } + + fn update_metadata(&self, _update: MetadataUpdate) -> VfsResult<()> { + Ok(()) + } + + fn filesystem(&self) -> &dyn FilesystemOps { + &TEST_FS + } + + fn sync(&self, _data_only: bool) -> VfsResult<()> { + Ok(()) + } + + fn into_any(self: Arc) -> Arc { + self + } + + fn flags(&self) -> NodeFlags { + NodeFlags::empty() + } +} + +#[cfg(feature = "ax-std")] +impl DirNodeOps for TestDir { + fn read_dir(&self, _offset: u64, _sink: &mut dyn DirEntrySink) -> VfsResult { + Ok(0) + } + + fn lookup(&self, name: &str) -> VfsResult { + if name == "new" && !self.renamed.load(Ordering::Acquire) { + return Err(VfsError::NotFound); + } + Ok(self.new_child(name)) + } + + fn create( + &self, + _name: &str, + _node_type: NodeType, + _permission: NodePermission, + ) -> VfsResult { + Err(VfsError::Unsupported) + } + + fn link(&self, _name: &str, _node: &DirEntry) -> VfsResult { + Err(VfsError::Unsupported) + } + + fn unlink(&self, _name: &str) -> VfsResult<()> { + Err(VfsError::Unsupported) + } + + fn rename(&self, _src_name: &str, _dst_dir: &DirNode, dst_name: &str) -> VfsResult<()> { + if dst_name == "new" { + self.renamed.store(true, Ordering::Release); + } + Ok(()) + } +} + +#[cfg(feature = "ax-std")] +fn vfs_cache_single_task_abba() { + let dir = TestDir::new_entry(1, "dir"); + + { + let _guard = dir.user_data(); + let _child = dir.as_dir().unwrap().lookup("child").unwrap(); + println!("vfs-cache-single: recorded dentry user_data -> dir cache"); + } + + dir.as_dir() + .unwrap() + .insert_cache("old".to_string(), TestDir::new_entry(2, "old")); + dir.as_dir() + .unwrap() + .rename("old", dir.as_dir().unwrap(), "new") + .unwrap(); +} + #[cfg(feature = "ax-std")] fn run_case(case: &str) { match case { @@ -258,6 +431,7 @@ fn run_case(case: &str) { "mixed-two-task" => mixed_two_task_abba(), "mixed-ms-single" => mixed_ms_single_task_abba(), "mixed-ms-two-task" => mixed_ms_two_task_abba(), + "vfs-cache-single" => vfs_cache_single_task_abba(), other => panic!("unsupported LOCKDEP_CASE: {other}"), } }