diff --git a/Cargo.lock b/Cargo.lock index b25cdf0bd1..8a85c06a38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -781,6 +781,7 @@ version = "0.5.23" dependencies = [ "ax-alloc", "ax-cpu", + "ax-crate-interface", "ax-kernel-guard", "ax-kspin", "ax-memory-addr", @@ -1144,9 +1145,9 @@ dependencies = [ "ax-memory-addr", "ax-mm", "ax-percpu", - "ax-sched", "ax-timer-list", "axpoll", + "bare-task", "cfg-if", "extern-trait", "futures-util", @@ -1567,6 +1568,13 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603" +[[package]] +name = "bare-task" +version = "0.1.0" +dependencies = [ + "trait-ffi", +] + [[package]] name = "base64" version = "0.13.1" diff --git a/Cargo.toml b/Cargo.toml index cd32905a26..2037885680 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "components/axtest/axtest_macros", "components/ax-lazyinit", "components/axbacktrace", + "components/bare-task", "components/axcpu", "components/axerrno", "components/axfs-ng-vfs", @@ -187,6 +188,7 @@ ax-timer-list = { version = "0.3.5", path = "components/timer_list" } axaddrspace = { version = "0.5.15", path = "virtualization/axaddrspace" } axbacktrace = { version = "0.4.4", path = "components/axbacktrace" } axbuild = { version = "0.4.16", path = "scripts/axbuild" } +bare-task = { version = "0.1.0", path = "components/bare-task" } axdevice = { version = "0.5.1", path = "virtualization/axdevice" } axdevice_base = { version = "0.6.0", path = "virtualization/axdevice_base" } axfs-ng-vfs = { version = "0.5.3", path = "components/axfs-ng-vfs" } diff --git a/apps/arceos/thread_test/Cargo.toml b/apps/arceos/thread_test/Cargo.toml index f7a8edd73b..ed91fb61f6 100644 --- a/apps/arceos/thread_test/Cargo.toml +++ b/apps/arceos/thread_test/Cargo.toml @@ -7,7 +7,7 @@ publish = false [features] default = [] -arceos = ["dep:ax-std", "ax-std/multitask", "ax-std/irq"] +arceos = ["dep:ax-std", "ax-std/multitask"] [dependencies] ax-std = { workspace = true, optional = true } diff --git a/apps/arceos/tokio_test/Cargo.toml b/apps/arceos/tokio_test/Cargo.toml index 527694c405..c260e747f2 100644 --- a/apps/arceos/tokio_test/Cargo.toml +++ b/apps/arceos/tokio_test/Cargo.toml @@ -8,7 +8,7 @@ publish = false [features] default = [] -arceos = ["dep:ax-std", "ax-std/multitask", "ax-std/irq"] +arceos = ["dep:ax-std", "ax-std/multitask"] [dependencies] ax-std = { workspace = true, optional = true } diff --git a/apps/starry/orangepi-5-plus-uvc-rknn/build-aarch64-unknown-none-softfloat.toml b/apps/starry/orangepi-5-plus-uvc-rknn/build-aarch64-unknown-none-softfloat.toml index 50d7a48504..fd974c5411 100644 --- a/apps/starry/orangepi-5-plus-uvc-rknn/build-aarch64-unknown-none-softfloat.toml +++ b/apps/starry/orangepi-5-plus-uvc-rknn/build-aarch64-unknown-none-softfloat.toml @@ -1,7 +1,6 @@ # Build-time config for the Orange Pi 5 Plus UVC + RKNN Starry example. target = "aarch64-unknown-none-softfloat" features = [ - "ax-driver/irq", "ax-driver/list-pci-devices", "ax-driver/rk3588-pcie", "ax-driver/realtek-rtl8125", diff --git a/components/axpoll/src/lib.rs b/components/axpoll/src/lib.rs index 8c789be5ef..06fc55ba29 100644 --- a/components/axpoll/src/lib.rs +++ b/components/axpoll/src/lib.rs @@ -5,7 +5,7 @@ extern crate alloc; -use alloc::{boxed::Box, vec::Vec}; +use alloc::vec::Vec; use core::{ mem::MaybeUninit, task::{Context, Waker}, @@ -64,7 +64,7 @@ pub trait Pollable { fn register(&self, context: &mut Context<'_>, events: IoEvents); } -const POLL_SET_CAPACITY: usize = 64; +const IRQ_WAKE_BATCH: usize = 64; struct Entry { waker: Waker, @@ -78,49 +78,33 @@ impl Entry { } struct Inner { - entries: Box<[MaybeUninit]>, - cursor: usize, + entries: Vec, } impl Inner { fn new() -> Self { Self { - entries: Box::new_uninit_slice(POLL_SET_CAPACITY), - cursor: 0, + entries: Vec::new(), } } - fn len(&self) -> usize { - self.cursor.min(POLL_SET_CAPACITY) - } - fn is_empty(&self) -> bool { - self.cursor == 0 + self.entries.is_empty() } - fn push_entry(&mut self, entry: Entry) { - debug_assert!(self.cursor < POLL_SET_CAPACITY); - let slot = self.cursor; - self.cursor += 1; - self.entries[slot].write(entry); - } + fn register(&mut self, waker: &Waker, interests: IoEvents) { + for entry in &mut self.entries { + if entry.waker.will_wake(waker) { + entry.waker = waker.clone(); + entry.interests |= interests; + return; + } + } - fn register(&mut self, waker: &Waker, interests: IoEvents) -> Option { - let slot = self.cursor % POLL_SET_CAPACITY; - let replaced = if self.cursor >= POLL_SET_CAPACITY { - let old = unsafe { self.entries[slot].assume_init_read() }; - let replaced = (!old.waker.will_wake(waker)).then_some(old); - self.cursor = ((slot + 1) % POLL_SET_CAPACITY) + POLL_SET_CAPACITY; - replaced - } else { - self.cursor += 1; - None - }; - self.entries[slot].write(Entry { + self.entries.push(Entry { waker: waker.clone(), interests, }); - replaced } fn drain_ready(&mut self, ready: IoEvents, ready_entries: &mut Vec) { @@ -128,25 +112,39 @@ impl Inner { return; } - let mut old = Self::new(); - core::mem::swap(&mut old, self); + let mut index = 0; + while index < self.entries.len() { + if self.entries[index].interests.intersects(ready) { + ready_entries.push(self.entries.swap_remove(index)); + } else { + index += 1; + } + } + } - for i in 0..old.len() { - let entry = unsafe { old.entries[i].assume_init_read() }; - if entry.interests.intersects(ready) { - ready_entries.push(entry); + fn drain_ready_batch( + &mut self, + ready: IoEvents, + ready_entries: &mut [MaybeUninit], + ) -> usize { + let mut ready_len = 0; + let mut index = 0; + while index < self.entries.len() && ready_len < ready_entries.len() { + if self.entries[index].interests.intersects(ready) { + ready_entries[ready_len].write(self.entries.swap_remove(index)); + ready_len += 1; } else { - self.push_entry(entry); + index += 1; } } - old.cursor = 0; + ready_len } } impl Drop for Inner { fn drop(&mut self) { - for i in 0..self.len() { - unsafe { self.entries[i].assume_init_read() }.wake(); + for entry in self.entries.drain(..) { + entry.wake(); } } } @@ -174,15 +172,10 @@ impl PollSet { /// from hard IRQ, NMI, or trap callbacks, and must not hold locks that may /// be re-entered by the registered waker or by poll wakeup paths. pub unsafe fn register(&self, waker: &Waker, interests: IoEvents) { - let replaced = { - self.0 - .call_once(|| SpinNoIrq::new(Inner::new())) - .lock() - .register(waker, interests) - }; - if let Some(entry) = replaced { - entry.wake(); - } + self.0 + .call_once(|| SpinNoIrq::new(Inner::new())) + .lock() + .register(waker, interests); } /// Wakes up registered wakers whose interests intersect `ready`. @@ -198,7 +191,7 @@ impl PollSet { let Some(inner) = self.0.get() else { return 0; }; - let mut ready_entries = Vec::with_capacity(POLL_SET_CAPACITY); + let mut ready_entries = Vec::new(); { inner.lock().drain_ready(ready, &mut ready_entries); } @@ -211,42 +204,34 @@ impl PollSet { /// Wakes up registered wakers whose interests intersect `ready` from IRQ context. /// - /// Unlike [`wake`](Self::wake), this does not allocate a replacement - /// waiter buffer. It drains the already-initialized entries in place, so - /// device IRQ handlers can acknowledge the device and then wake matching - /// poll waiters without allocating in hard IRQ context. + /// This method is kept for legacy users only. New hard IRQ code should + /// publish state into the device/backend and wake a deferred worker via an + /// IRQ-safe task waker instead. Unlike [`wake`](Self::wake), this method + /// avoids allocation by draining matching entries in fixed-size batches. pub fn wake_from_irq(&self, ready: IoEvents) -> usize { let Some(inner) = self.0.get() else { return 0; }; - let mut ready_entries = [const { MaybeUninit::::uninit() }; POLL_SET_CAPACITY]; - let ready_len = { - let mut inner = inner.lock(); - let len = inner.len(); - if len == 0 { - return 0; - } - let mut ready_len = 0; - let mut keep_len = 0; - for i in 0..len { - let entry = unsafe { inner.entries[i].assume_init_read() }; - if entry.interests.intersects(ready) { - ready_entries[ready_len].write(entry); - ready_len += 1; - } else { - inner.entries[keep_len].write(entry); - keep_len += 1; + let mut woke = 0; + loop { + let mut ready_entries = [const { MaybeUninit::::uninit() }; IRQ_WAKE_BATCH]; + let ready_len = { + let mut inner = inner.lock(); + if inner.is_empty() { + return woke; } + inner.drain_ready_batch(ready, &mut ready_entries) + }; + if ready_len == 0 { + return woke; } - inner.cursor = keep_len; - ready_len - }; - for entry in ready_entries.iter_mut().take(ready_len) { - unsafe { entry.assume_init_read() }.wake(); + woke += ready_len; + for entry in ready_entries.iter_mut().take(ready_len) { + unsafe { entry.assume_init_read() }.wake(); + } } - ready_len } } diff --git a/components/axpoll/tests/tests.rs b/components/axpoll/tests/tests.rs index 4ec16e3b14..10741fb0c8 100644 --- a/components/axpoll/tests/tests.rs +++ b/components/axpoll/tests/tests.rs @@ -104,35 +104,6 @@ fn wake_runs_wakers_after_releasing_pollset_lock() { assert_eq!(unsafe { ps.wake(IoEvents::OUT) }, 1); } -#[test] -fn register_overwrite_wakes_after_releasing_pollset_lock() { - let ps = Arc::new(PollSet::new()); - let (started_tx, started_rx) = mpsc::channel(); - let (done_tx, done_rx) = mpsc::channel(); - let reentrant = Arc::new(ReentrantRegister { - poll: ps.clone(), - interests: IoEvents::OUT, - started: started_tx, - done: done_tx, - }); - let reentrant_waker = Waker::from(reentrant); - unsafe { ps.register(&reentrant_waker, IoEvents::IN) }; - for _ in 1..64 { - let waker = Waker::from(Counter::new()); - unsafe { ps.register(&waker, IoEvents::IN) }; - } - - let register_ps = ps.clone(); - let register_thread = thread::spawn(move || { - let waker = Waker::from(Counter::new()); - unsafe { register_ps.register(&waker, IoEvents::IN) }; - }); - - assert_reentrant_wake_completed(started_rx, done_rx); - register_thread.join().unwrap(); - assert_eq!(unsafe { ps.wake(IoEvents::OUT) }, 1); -} - #[test] fn empty_return() { let ps = PollSet::new(); @@ -275,7 +246,7 @@ fn irq_wake_drains_without_double_wake() { } #[test] -fn full_capacity() { +fn repeated_same_waker_register_is_idempotent() { let ps = PollSet::new(); let counter = Counter::new(); for _ in 0..64 { @@ -284,12 +255,26 @@ fn full_capacity() { unsafe { ps.register(cx.waker(), IoEvents::IN) }; } let woke = unsafe { ps.wake(IoEvents::IN) }; - assert_eq!(woke, 64); - assert_eq!(counter.count(), 64); + assert_eq!(woke, 1); + assert_eq!(counter.count(), 1); +} + +#[test] +fn repeated_same_waker_register_merges_interests() { + let ps = PollSet::new(); + let counter = Counter::new(); + let w = Waker::from(counter.clone()); + unsafe { + ps.register(&w, IoEvents::IN); + ps.register(&w, IoEvents::OUT); + } + + assert_eq!(unsafe { ps.wake(IoEvents::IN) }, 1); + assert_eq!(counter.count(), 1); } #[test] -fn overwrite() { +fn more_than_legacy_capacity_keeps_all_waiters_until_ready() { let ps = PollSet::new(); let counters = (0..65).map(|_| Counter::new()).collect::>(); for c in &counters { @@ -297,11 +282,26 @@ fn overwrite() { let cx = Context::from_waker(&w); unsafe { ps.register(cx.waker(), IoEvents::IN) }; } - assert_eq!(unsafe { ps.wake(IoEvents::IN) }, 64); + assert!(counters.iter().all(|counter| counter.count() == 0)); + assert_eq!(unsafe { ps.wake(IoEvents::IN) }, 65); let total: usize = counters.iter().map(|c| c.count()).sum(); assert_eq!(total, 65); } +#[test] +fn irq_wake_batches_more_than_legacy_capacity() { + let ps = PollSet::new(); + let counters = (0..130).map(|_| Counter::new()).collect::>(); + for c in &counters { + let w = Waker::from(c.clone()); + unsafe { ps.register(&w, IoEvents::IN) }; + } + + assert_eq!(ps.wake_from_irq(IoEvents::IN), 130); + assert!(counters.iter().all(|counter| counter.count() == 1)); + assert_eq!(ps.wake_from_irq(IoEvents::IN), 0); +} + #[test] fn drop_wakes() { let ps = PollSet::new(); @@ -312,5 +312,5 @@ fn drop_wakes() { unsafe { ps.register(cx.waker(), IoEvents::IN) }; } drop(ps); - assert_eq!(counters.count(), 10); + assert_eq!(counters.count(), 1); } diff --git a/components/bare-task/Cargo.toml b/components/bare-task/Cargo.toml new file mode 100644 index 0000000000..fe88080f1b --- /dev/null +++ b/components/bare-task/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "bare-task" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +description = "OS-independent task scheduling core with trait-ffi host hooks" + +[features] +default = [] +std = [] +host-test = ["std"] +smp = [] +ipi = ["smp"] +irq = [] + +[dependencies] +trait-ffi = "0.2.4" diff --git a/components/bare-task/src/core.rs b/components/bare-task/src/core.rs new file mode 100644 index 0000000000..202e615489 --- /dev/null +++ b/components/bare-task/src/core.rs @@ -0,0 +1,437 @@ +//! Core task state shared by schedulers and wake paths. + +use alloc::sync::Arc; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU8, AtomicU64, AtomicUsize, Ordering}; + +/// CPU identifier used by the bare task core. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct CpuId(pub usize); + +/// Fixed-width CPU mask for the core scheduler protocols. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CpuMask(u64); + +impl CpuMask { + /// Creates a mask with one CPU enabled. + pub const fn one(cpu: CpuId) -> Self { + Self(1u64 << cpu.0) + } + + /// Creates a mask with the lowest `count` CPUs enabled. + pub const fn first(count: usize) -> Self { + if count >= 64 { + Self(u64::MAX) + } else { + Self((1u64 << count) - 1) + } + } + + /// Creates a CPU mask from raw bits. + pub const fn from_bits(bits: u64) -> Self { + Self(bits) + } + + /// Returns whether the mask contains `cpu`. + pub const fn contains(self, cpu: CpuId) -> bool { + (self.0 & (1u64 << cpu.0)) != 0 + } + + /// Returns the raw mask bits. + pub const fn bits(self) -> u64 { + self.0 + } +} + +/// Unique task identifier. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct TaskId(pub u64); + +impl TaskId { + /// Allocates a fresh task id. + pub fn allocate() -> Self { + static ID_COUNTER: AtomicU64 = AtomicU64::new(1); + Self(ID_COUNTER.fetch_add(1, Ordering::Relaxed)) + } + + /// Returns the raw numeric id. + pub const fn as_u64(self) -> u64 { + self.0 + } +} + +/// Core task scheduling states. +#[repr(u8)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TaskState { + /// Task is currently running. + Running = 1, + /// Task is ready to run. + Ready = 2, + /// Task is blocked on an event. + Blocked = 3, + /// Task has exited. + Exited = 4, +} + +impl TaskState { + fn from_raw(raw: u8) -> Self { + match raw { + 1 => Self::Running, + 2 => Self::Ready, + 3 => Self::Blocked, + 4 => Self::Exited, + _ => Self::Exited, + } + } +} + +/// Shared task reference used by the bare task core. +pub type TaskRef = Arc; + +/// OS-independent task state. +pub struct TaskCore { + id: TaskId, + state: AtomicU8, + cpu_id: AtomicUsize, + cpumask: AtomicU64, + on_cpu: AtomicBool, + wait_queue_key: AtomicUsize, + timer_ticket_id: AtomicU64, + preempt_pending: AtomicBool, + force_resched_pending: AtomicBool, + preempt_disable_count: AtomicUsize, + interrupted: AtomicBool, + wake_pending: AtomicBool, + wake_seq: AtomicU64, + wake_bits: AtomicU64, + wake_generation: AtomicU64, + wake_next: AtomicPtr<()>, +} + +impl TaskCore { + /// Creates a task core in [`TaskState::Ready`]. + pub fn new(id: TaskId, cpu_id: CpuId) -> Self { + Self { + id, + state: AtomicU8::new(TaskState::Ready as u8), + cpu_id: AtomicUsize::new(cpu_id.0), + cpumask: AtomicU64::new(CpuMask::one(cpu_id).bits()), + on_cpu: AtomicBool::new(false), + wait_queue_key: AtomicUsize::new(0), + timer_ticket_id: AtomicU64::new(0), + preempt_pending: AtomicBool::new(false), + force_resched_pending: AtomicBool::new(false), + preempt_disable_count: AtomicUsize::new(0), + interrupted: AtomicBool::new(false), + wake_pending: AtomicBool::new(false), + wake_seq: AtomicU64::new(0), + wake_bits: AtomicU64::new(0), + wake_generation: AtomicU64::new(1), + wake_next: AtomicPtr::new(core::ptr::null_mut()), + } + } + + /// Returns the task id. + pub const fn id(&self) -> TaskId { + self.id + } + + /// Returns the current state. + pub fn state(&self) -> TaskState { + TaskState::from_raw(self.state.load(Ordering::Acquire)) + } + + /// Stores a new state. + pub fn set_state(&self, state: TaskState) { + self.state.store(state as u8, Ordering::Release); + } + + /// Performs a state transition if the task is currently in `from`. + pub fn transition_state(&self, from: TaskState, to: TaskState) -> bool { + self.state + .compare_exchange(from as u8, to as u8, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + + /// Returns the task's last selected CPU. + pub fn cpu_id(&self) -> CpuId { + CpuId(self.cpu_id.load(Ordering::Acquire)) + } + + /// Sets the task's last selected CPU. + pub fn set_cpu_id(&self, cpu_id: CpuId) { + self.cpu_id.store(cpu_id.0, Ordering::Release); + } + + /// Sets the CPU affinity mask. + pub fn set_cpumask(&self, cpumask: CpuMask) { + self.cpumask.store(cpumask.bits(), Ordering::Release); + } + + /// Returns the CPU affinity mask. + pub fn cpumask(&self) -> CpuMask { + CpuMask(self.cpumask.load(Ordering::Acquire)) + } + + /// Returns whether the task is marked as present in a wait queue. + pub fn in_wait_queue(&self) -> bool { + self.wait_queue_key.load(Ordering::Acquire) != 0 + } + + /// Updates wait-queue membership. + pub fn set_in_wait_queue(&self, in_wait_queue: bool) { + self.wait_queue_key.store( + if in_wait_queue { usize::MAX } else { 0 }, + Ordering::Release, + ); + } + + /// Returns the wait queue key this task is currently linked to, or zero. + pub fn wait_queue_key(&self) -> usize { + self.wait_queue_key.load(Ordering::Acquire) + } + + /// Returns whether this task is currently linked to `wait_queue_key`. + pub fn is_waiting_on(&self, wait_queue_key: usize) -> bool { + wait_queue_key != 0 && self.wait_queue_key() == wait_queue_key + } + + /// Marks this task as linked to the wait queue identified by `wait_queue_key`. + pub fn set_wait_queue_key(&self, wait_queue_key: usize) { + assert!(wait_queue_key != 0); + self.wait_queue_key.store(wait_queue_key, Ordering::Release); + } + + /// Clears the wait queue key only if it still matches `wait_queue_key`. + pub fn clear_wait_queue_key(&self, wait_queue_key: usize) -> bool { + self.wait_queue_key + .compare_exchange(wait_queue_key, 0, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + + /// Clears any current wait queue membership and returns its previous key. + pub fn take_wait_queue_key(&self) -> usize { + self.wait_queue_key.swap(0, Ordering::AcqRel) + } + + /// Returns whether this task is still finishing a context switch on a CPU. + pub fn on_cpu(&self) -> bool { + self.on_cpu.load(Ordering::Acquire) + } + + /// Updates the context-switch ownership marker. + pub fn set_on_cpu(&self, on_cpu: bool) { + self.on_cpu.store(on_cpu, Ordering::Release); + } + + /// Returns the currently armed timer ticket. + pub fn timer_ticket(&self) -> u64 { + self.timer_ticket_id.load(Ordering::Acquire) + } + + /// Stores a non-zero timer ticket. + pub fn set_timer_ticket(&self, timer_ticket_id: u64) { + assert!(timer_ticket_id != 0); + self.timer_ticket_id + .store(timer_ticket_id, Ordering::Release); + } + + /// Marks the current timer ticket as expired/cancelled. + pub fn timer_ticket_expired(&self) { + self.timer_ticket_id.store(0, Ordering::Release); + } + + /// Sets ordinary preemption pending state. + pub fn set_preempt_pending(&self, pending: bool) { + self.preempt_pending.store(pending, Ordering::Release); + } + + /// Returns ordinary preemption pending state. + pub fn preempt_pending(&self) -> bool { + self.preempt_pending.load(Ordering::Acquire) + } + + /// Sets force-reschedule pending state. + pub fn set_force_resched_pending(&self, pending: bool) { + self.force_resched_pending.store(pending, Ordering::Release); + } + + /// Returns force-reschedule pending state. + pub fn force_resched_pending(&self) -> bool { + self.force_resched_pending.load(Ordering::Acquire) + } + + /// Atomically consumes force-reschedule pending state. + pub fn take_force_resched_pending(&self) -> bool { + self.force_resched_pending.swap(false, Ordering::AcqRel) + } + + /// Returns the preemption disable nesting depth. + pub fn preempt_count(&self) -> usize { + self.preempt_disable_count.load(Ordering::Acquire) + } + + /// Returns whether preemption is allowed for the expected guard depth. + pub fn can_preempt(&self, current_disable_count: usize) -> bool { + self.preempt_count() == current_disable_count + } + + /// Increments the preemption disable nesting depth. + pub fn disable_preempt(&self) { + self.preempt_disable_count.fetch_add(1, Ordering::Release); + } + + /// Decrements the preemption disable nesting depth and returns true if it reached zero. + pub fn enable_preempt(&self) -> bool { + self.preempt_disable_count.fetch_sub(1, Ordering::Release) == 1 + } + + /// Returns whether the task has a pending interrupt. + pub fn interrupted(&self) -> bool { + self.interrupted.load(Ordering::Acquire) + } + + /// Publishes an interrupt to the task. + pub fn interrupt(&self) { + self.interrupted.store(true, Ordering::Release); + } + + /// Clears the interrupt flag. + pub fn clear_interrupt(&self) { + self.interrupted.store(false, Ordering::Release); + } + + /// Atomically consumes the interrupt flag. + pub fn take_interrupt(&self) -> bool { + self.interrupted.swap(false, Ordering::AcqRel) + } + + /// Publishes coalesced wake bits. + pub fn publish_wake_bits(&self, bits: u64) { + if bits != 0 { + self.wake_bits.fetch_or(bits, Ordering::AcqRel); + } + } + + /// Bumps the wake sequence and returns the new value. + pub fn bump_wake_seq(&self) -> u64 { + self.wake_seq.fetch_add(1, Ordering::AcqRel) + 1 + } + + /// Returns the current wake sequence. + pub fn wake_seq(&self) -> u64 { + self.wake_seq.load(Ordering::Acquire) + } + + /// Takes coalesced wake bits. + pub fn take_wake_bits(&self) -> u64 { + self.wake_bits.swap(0, Ordering::AcqRel) + } + + /// Marks the task pending for hard-IRQ wake queue insertion. + pub fn mark_wake_pending(&self) -> bool { + !self.wake_pending.swap(true, Ordering::AcqRel) + } + + /// Clears and returns the previous pending bit. + pub fn take_wake_pending(&self) -> bool { + self.wake_pending.swap(false, Ordering::AcqRel) + } + + /// Returns the wake generation. + pub fn wake_generation(&self) -> u64 { + self.wake_generation.load(Ordering::Acquire) + } + + /// Returns whether `generation` still matches this task. + pub fn wake_generation_matches(&self, generation: u64) -> bool { + self.wake_generation() == generation && self.state() != TaskState::Exited + } + + /// Expires old wake handles. + pub fn expire_wakers(&self) { + self.wake_generation.fetch_add(1, Ordering::AcqRel); + self.wake_pending.store(false, Ordering::Release); + self.wake_bits.store(0, Ordering::Release); + self.wake_next + .store(core::ptr::null_mut(), Ordering::Release); + } + + /// Stores the intrusive IRQ wake link. + pub fn set_wake_next(&self, next: *mut T) { + self.wake_next.store(next.cast::<()>(), Ordering::Release); + } + + /// Loads the intrusive IRQ wake link. + pub fn wake_next(&self) -> *mut T { + self.wake_next.load(Ordering::Acquire).cast::() + } + + /// Clears the intrusive IRQ wake link. + pub fn clear_wake_link(&self) { + self.wake_next + .store(core::ptr::null_mut(), Ordering::Release); + } +} + +#[cfg(test)] +mod tests { + use super::{CpuId, CpuMask, TaskCore, TaskId, TaskState}; + + #[test] + fn task_core_owns_scheduler_and_wake_metadata() { + let task = TaskCore::new(TaskId(7), CpuId(0)); + + assert_eq!(task.state(), TaskState::Ready); + assert!(task.transition_state(TaskState::Ready, TaskState::Running)); + assert_eq!(task.state(), TaskState::Running); + + task.set_cpu_id(CpuId(2)); + task.set_cpumask(CpuMask::one(CpuId(2))); + assert_eq!(task.cpu_id(), CpuId(2)); + assert!(task.cpumask().contains(CpuId(2))); + + assert!(!task.in_wait_queue()); + task.set_in_wait_queue(true); + assert!(task.in_wait_queue()); + + task.set_timer_ticket(11); + assert_eq!(task.timer_ticket(), 11); + task.timer_ticket_expired(); + assert_eq!(task.timer_ticket(), 0); + + task.set_on_cpu(true); + assert!(task.on_cpu()); + task.set_on_cpu(false); + assert!(!task.on_cpu()); + + task.set_preempt_pending(true); + task.set_force_resched_pending(true); + assert!(task.preempt_pending()); + assert!(task.force_resched_pending()); + assert_eq!(task.preempt_count(), 0); + task.disable_preempt(); + assert_eq!(task.preempt_count(), 1); + assert!(!task.can_preempt(0)); + task.enable_preempt(); + assert!(task.can_preempt(0)); + assert!(task.take_force_resched_pending()); + + assert!(!task.interrupted()); + task.interrupt(); + assert!(task.interrupted()); + assert!(task.take_interrupt()); + assert!(!task.interrupted()); + + task.publish_wake_bits(0b0011); + task.publish_wake_bits(0b0100); + assert_eq!(task.take_wake_bits(), 0b0111); + assert_eq!(task.bump_wake_seq(), 1); + assert_eq!(task.wake_seq(), 1); + assert!(task.mark_wake_pending()); + assert!(!task.mark_wake_pending()); + assert!(task.take_wake_pending()); + } +} diff --git a/components/bare-task/src/cpu.rs b/components/bare-task/src/cpu.rs new file mode 100644 index 0000000000..455fcef975 --- /dev/null +++ b/components/bare-task/src/cpu.rs @@ -0,0 +1,421 @@ +//! Per-CPU task runtime core. + +use alloc::{sync::Arc, vec::Vec}; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +use crate::{ + BaseScheduler, CFSTask, FifoTask, IrqWakeQueueCore, RRTask, RunQueueCore, ScheduledTask, + TaskCore, TaskOps, TaskRef, TaskState, WakeResult, + wake::{HardIrqWaker, WakeBits}, +}; + +/// Runtime IPI event kind. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IpiEvent { + /// Re-evaluate scheduling on the target CPU. + Reschedule, + /// Drain the target CPU hard-IRQ wake queue. + IrqWakeDrain, + /// Run timer service work on the target CPU. + TimerService, +} + +impl IpiEvent { + const fn bit(self) -> usize { + match self { + Self::Reschedule => 1 << 0, + Self::IrqWakeDrain => 1 << 1, + Self::TimerService => 1 << 2, + } + } +} + +/// Coalesced runtime IPI event set. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct IpiEvents(usize); + +impl IpiEvents { + /// Creates an empty event set. + pub const fn empty() -> Self { + Self(0) + } + + /// Creates an event set from explicit events. + pub fn from_events(events: &[IpiEvent]) -> Self { + let mut bits = 0; + for event in events { + bits |= event.bit(); + } + Self(bits) + } + + /// Returns whether the set is empty. + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + /// Returns whether `event` is present. + pub const fn contains(self, event: IpiEvent) -> bool { + self.0 & event.bit() != 0 + } + + /// Returns raw event bits. + pub const fn bits(self) -> usize { + self.0 + } + + /// Returns the number of pending events. + pub const fn count(self) -> usize { + self.0.count_ones() as usize + } +} + +/// Scheduler item that can be created from a bare [`TaskRef`]. +pub trait ReadyTaskFromCore: ScheduledTask { + /// Creates a scheduler item for `task`. + fn from_task_ref(task: TaskRef) -> Self; +} + +impl ReadyTaskFromCore for Arc> { + fn from_task_ref(task: TaskRef) -> Self { + Arc::new(FifoTask::new(task)) + } +} + +impl ReadyTaskFromCore for Arc> { + fn from_task_ref(task: TaskRef) -> Self { + Arc::new(RRTask::new(task)) + } +} + +impl ReadyTaskFromCore for Arc> { + fn from_task_ref(task: TaskRef) -> Self { + Arc::new(CFSTask::new(task)) + } +} + +struct TimerEntry { + deadline_nanos: u64, + task: TaskRef, + active: bool, +} + +/// Per-CPU task runtime core. +pub struct BareCpuCore +where + S::SchedItem: ScheduledTask, +{ + cpu_id: crate::CpuId, + run_queue: crate::sync::SpinMutex>, + irq_wake_queue: IrqWakeQueueCore, + ipi_pending: AtomicUsize, + timer_service_pending: AtomicBool, + ready_count: AtomicUsize, + current_task: AtomicPtr<()>, + idle_task: AtomicPtr<()>, + init_task: AtomicPtr<()>, + prev_task: AtomicPtr<()>, + timers: crate::sync::SpinMutex>, +} + +impl BareCpuCore +where + S: BaseScheduler, + S::SchedItem: ScheduledTask, +{ + /// Creates a CPU core with an empty run queue. + pub fn new(cpu_id: crate::CpuId, scheduler: S) -> Self { + Self { + cpu_id, + run_queue: crate::sync::SpinMutex::new(RunQueueCore::new(scheduler)), + irq_wake_queue: IrqWakeQueueCore::new(), + ipi_pending: AtomicUsize::new(0), + timer_service_pending: AtomicBool::new(false), + ready_count: AtomicUsize::new(0), + current_task: AtomicPtr::new(core::ptr::null_mut()), + idle_task: AtomicPtr::new(core::ptr::null_mut()), + init_task: AtomicPtr::new(core::ptr::null_mut()), + prev_task: AtomicPtr::new(core::ptr::null_mut()), + timers: crate::sync::SpinMutex::new(Vec::new()), + } + } + + /// Returns this CPU id. + pub const fn cpu_id(&self) -> crate::CpuId { + self.cpu_id + } + + /// Runs `f` with mutable access to this CPU run queue core. + pub fn with_run_queue(&self, f: impl FnOnce(&mut RunQueueCore) -> R) -> R { + f(&mut self.run_queue.lock()) + } + + /// Returns the number of tasks inserted through this runtime core. + pub fn run_queue_len(&self) -> usize { + self.ready_count.load(Ordering::Acquire) + } + + /// Requests an IPI event, returning true if it newly set a pending bit. + pub fn request_ipi(&self, event: IpiEvent) -> bool { + let bit = event.bit(); + self.ipi_pending.fetch_or(bit, Ordering::AcqRel) & bit == 0 + } + + /// Takes coalesced pending IPI events. + pub fn take_pending_ipis(&self) -> IpiEvents { + IpiEvents(self.ipi_pending.swap(0, Ordering::AcqRel)) + } + + /// Clears one pending IPI event. + pub fn clear_ipi(&self, event: IpiEvent) { + self.ipi_pending.fetch_and(!event.bit(), Ordering::AcqRel); + } + + /// Returns whether timer service is pending. + pub fn timer_service_pending(&self) -> bool { + self.timer_service_pending.load(Ordering::Acquire) + } + + /// Marks timer service pending. + pub fn mark_timer_service_pending(&self) -> bool { + !self.timer_service_pending.swap(true, Ordering::AcqRel) + } + + /// Clears timer service pending. + pub fn clear_timer_service_pending(&self) { + self.timer_service_pending.store(false, Ordering::Release); + } + + /// Installs the current task raw pointer. + pub fn set_current_task_ptr(&self, ptr: *mut ()) { + self.current_task.store(ptr, Ordering::Release); + } + + /// Returns the current task raw pointer. + pub fn current_task_ptr(&self) -> *mut () { + self.current_task.load(Ordering::Acquire) + } + + /// Installs the idle task raw pointer. + pub fn set_idle_task_ptr(&self, ptr: *mut ()) { + self.idle_task.store(ptr, Ordering::Release); + } + + /// Installs the init task raw pointer. + pub fn set_init_task_ptr(&self, ptr: *mut ()) { + self.init_task.store(ptr, Ordering::Release); + } + + /// Stores the previous task raw pointer. + pub fn set_prev_task_ptr(&self, ptr: *mut ()) { + self.prev_task.store(ptr, Ordering::Release); + } + + /// Takes the previous task raw pointer. + pub fn take_prev_task_ptr(&self) -> *mut () { + self.prev_task.swap(core::ptr::null_mut(), Ordering::AcqRel) + } + + fn push_irq_wake_task(&self, task: &TaskRef) { + let task_ptr = Arc::as_ptr(task).cast_mut(); + // Keep the task alive until the drain path reconstructs the Arc. + unsafe { Arc::increment_strong_count(task_ptr) }; + self.irq_wake_queue.push(task_ptr.cast::<()>(), |next| { + task.set_wake_next(next.cast::()); + }); + } + + fn pop_irq_wake_task(&self) -> Option { + let head = self.irq_wake_queue.pop(|node| { + let task = unsafe { &*node.cast::() }; + task.wake_next::().cast::<()>() + })?; + Some(unsafe { Arc::from_raw(head.cast::()) }) + } + + fn add_timer(&self, deadline_nanos: u64, task: TaskRef) { + self.timers.lock().push(TimerEntry { + deadline_nanos, + task, + active: true, + }); + } + + fn take_expired_timer_tasks(&self, now_nanos: u64) -> Vec { + let mut expired = Vec::new(); + for timer in self.timers.lock().iter_mut() { + if timer.active && timer.deadline_nanos <= now_nanos { + timer.active = false; + expired.push(timer.task.clone()); + } + } + expired + } +} + +impl BareCpuCore +where + S: BaseScheduler, + S::SchedItem: ReadyTaskFromCore, +{ + fn enqueue_task_ref(&self, task: TaskRef) { + self.run_queue + .lock() + .add_task(S::SchedItem::from_task_ref(task)); + self.ready_count.fetch_add(1, Ordering::AcqRel); + } +} + +/// Multi-CPU task runtime core. +pub struct BareTaskRuntime +where + S::SchedItem: ScheduledTask, +{ + cpus: crate::sync::SpinMutex>>>, +} + +impl BareTaskRuntime +where + S: BaseScheduler, + S::SchedItem: ScheduledTask, +{ + /// Creates an empty runtime. + pub const fn new() -> Self { + Self { + cpus: crate::sync::SpinMutex::new(Vec::new()), + } + } + + /// Adds a CPU runtime core. + pub fn add_cpu(&self, cpu_id: crate::CpuId, scheduler: S) -> Arc> { + let cpu = Arc::new(BareCpuCore::new(cpu_id, scheduler)); + let mut cpus = self.cpus.lock(); + if cpus.len() == cpu_id.0 { + cpus.push(cpu.clone()); + } else if cpus.len() > cpu_id.0 { + cpus[cpu_id.0] = cpu.clone(); + } else { + panic!("CPU runtime cores must be installed in cpu-id order"); + } + cpu + } + + /// Returns a CPU runtime core. + pub fn cpu(&self, cpu_id: crate::CpuId) -> Option>> { + self.cpus.lock().get(cpu_id.0).cloned() + } + + fn cpu_expect(&self, cpu_id: crate::CpuId) -> Arc> { + self.cpu(cpu_id) + .expect("bare task CPU core is not installed") + } +} + +impl Default for BareTaskRuntime +where + S: BaseScheduler, + S::SchedItem: ScheduledTask, +{ + fn default() -> Self { + Self::new() + } +} + +impl BareTaskRuntime +where + S: BaseScheduler, + S::SchedItem: ReadyTaskFromCore, +{ + /// Publishes a hard IRQ wake and queues the task on its target CPU. + pub fn wake_from_irq(&self, waker: &HardIrqWaker, bits: WakeBits) -> WakeResult { + let (task, result) = waker.wake_from_irq(bits); + let Some(task) = task else { + return result; + }; + if result.woke() { + let cpu = self.cpu_expect(task.cpu_id()); + cpu.push_irq_wake_task(&task); + cpu.request_ipi(IpiEvent::IrqWakeDrain); + return WakeResult::new(true, false, true); + } + result + } + + /// Drains one CPU's hard IRQ wake queue into the run queue. + pub fn drain_irq_wake_queue(&self, cpu_id: crate::CpuId) -> usize { + let cpu = self.cpu_expect(cpu_id); + let mut drained = 0; + while let Some(task) = cpu.pop_irq_wake_task() { + task.clear_wake_link(); + if !task.take_wake_pending() { + continue; + } + if task.transition_state(TaskState::Blocked, TaskState::Ready) { + cpu.enqueue_task_ref(task); + drained += 1; + } + } + drained + } + + /// Enqueues an already-ready task on the target CPU. + pub fn add_ready_task(&self, cpu_id: crate::CpuId, task: TaskRef) { + self.cpu_expect(cpu_id).enqueue_task_ref(task); + } + + /// Adds a task timer to a CPU runtime. + pub fn add_task_timer(&self, cpu_id: crate::CpuId, deadline_nanos: u64, task: TaskRef) { + self.cpu_expect(cpu_id).add_timer(deadline_nanos, task); + } + + /// Handles a timer interrupt. This only marks deferred timer service work. + pub fn on_timer_irq(&self, cpu_id: crate::CpuId, _now_nanos: u64) { + let cpu = self.cpu_expect(cpu_id); + cpu.mark_timer_service_pending(); + } + + /// Handles a runtime IPI interrupt. + pub fn on_ipi_irq(&self, cpu_id: crate::CpuId) -> IpiEvents { + let events = self.cpu_expect(cpu_id).take_pending_ipis(); + if events.contains(IpiEvent::IrqWakeDrain) { + self.drain_irq_wake_queue(cpu_id); + } + events + } + + /// Drains expired timer tasks in task/deferred context. + pub fn drain_timer_service(&self, cpu_id: crate::CpuId, now_nanos: u64) -> usize { + let cpu = self.cpu_expect(cpu_id); + if !cpu.timer_service_pending() { + return 0; + } + cpu.clear_timer_service_pending(); + let expired = cpu.take_expired_timer_tasks(now_nanos); + let mut drained = 0; + for task in expired { + if task.transition_state(TaskState::Blocked, TaskState::Ready) { + cpu.enqueue_task_ref(task); + drained += 1; + } + } + drained + } +} + +impl TaskOps for TaskRef { + fn core(&self) -> &TaskCore { + self + } + + fn id_name(&self) -> alloc::string::String { + alloc::format!("task#{}", self.id().as_u64()) + } + + fn is_idle(&self) -> bool { + false + } + + fn is_init(&self) -> bool { + false + } +} diff --git a/components/bare-task/src/host.rs b/components/bare-task/src/host.rs new file mode 100644 index 0000000000..1ef6df4fde --- /dev/null +++ b/components/bare-task/src/host.rs @@ -0,0 +1,517 @@ +//! Deterministic host virtual SMP and IRQ runtime for tests. + +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use core::sync::atomic::{AtomicU64, Ordering}; + +use crate::{ + BareCpuCore, BareTaskRuntime, CpuId, CpuMask, FifoScheduler, HardIrqWaker, IpiEvent, TaskCore, + TaskId, TaskRef, TaskState, WakeResult, +}; + +/// Virtual IRQ identifier. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct IrqId(pub usize); + +/// Virtual IRQ trigger mode. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IrqMode { + /// Pending edge events coalesce while the line is pending. + Edge, + /// A level line remains pending until explicitly acknowledged. + Level, +} + +/// Virtual CPU register state observable by tests. +#[derive(Clone, Debug)] +pub struct VirtualRegisters { + /// CPU represented by this host worker. + pub cpu_id: CpuId, + /// Current task on this CPU. + pub current_task: Option, + /// Local IRQ enable flag. + pub irq_enabled: bool, + /// Whether this CPU is executing a hard IRQ callback. + pub in_irq: bool, + /// Preemption disable depth. + pub preempt_depth: usize, + /// Whether scheduling should be re-evaluated after IRQ exit. + pub need_resched: bool, +} + +impl VirtualRegisters { + fn new(cpu_id: CpuId) -> Self { + Self { + cpu_id, + current_task: None, + irq_enabled: true, + in_irq: false, + preempt_depth: 0, + need_resched: false, + } + } +} + +/// Virtual CPU state. +pub struct HostCpu { + /// Virtual registers. + pub regs: VirtualRegisters, + core: Arc>>, + delivered_ipis: usize, + delivered_irqs: usize, +} + +impl HostCpu { + fn new(cpu_id: CpuId, core: Arc>>) -> Self { + Self { + regs: VirtualRegisters::new(cpu_id), + core, + delivered_ipis: 0, + delivered_irqs: 0, + } + } + + /// Returns the number of runnable tasks queued on this CPU. + pub fn run_queue_len(&self) -> usize { + self.core.run_queue_len() + } + + /// Returns the number of delivered virtual IRQ callbacks. + pub fn delivered_irqs(&self) -> usize { + self.delivered_irqs + } + + /// Returns the number of delivered virtual IPI events. + pub fn delivered_ipis(&self) -> usize { + self.delivered_ipis + } +} + +type IrqCallback = Box; + +struct IrqLine { + enabled: bool, + pending: bool, + mode: IrqMode, + affinity: CpuMask, + callback: Option, +} + +struct TimerEvent { + id: u64, + cpu: CpuId, + deadline: u64, + task: TaskRef, + active: bool, +} + +/// Deterministic virtual SMP runtime. +pub struct HostSmpRuntime { + task_runtime: BareTaskRuntime>, + cpus: Vec, + irqs: Vec, + now_nanos: u64, + next_task_id: AtomicU64, + next_timer_id: u64, + timers: Vec, + ipi_irq: IrqId, + timer_irq: IrqId, +} + +impl HostSmpRuntime { + /// Creates a virtual runtime with `cpu_count` CPUs. + pub fn new(cpu_count: usize) -> Self { + assert!(cpu_count > 0); + let task_runtime = BareTaskRuntime::new(); + let cpus = (0..cpu_count) + .map(|cpu| { + let cpu_id = CpuId(cpu); + let core = task_runtime.add_cpu(cpu_id, FifoScheduler::new()); + HostCpu::new(cpu_id, core) + }) + .collect(); + let mut runtime = Self { + task_runtime, + cpus, + irqs: Vec::new(), + now_nanos: 0, + next_task_id: AtomicU64::new(1), + next_timer_id: 1, + timers: Vec::new(), + ipi_irq: IrqId(usize::MAX), + timer_irq: IrqId(usize::MAX), + }; + runtime.ipi_irq = + runtime.register_irq(IrqMode::Edge, CpuMask::first(cpu_count), |rt, cpu, _| { + rt.drain_ipi_queue(cpu); + }); + runtime.timer_irq = + runtime.register_irq(IrqMode::Edge, CpuMask::first(cpu_count), |rt, cpu, _| { + rt.drain_expired_timers(cpu); + }); + runtime + } + + /// Returns an immutable CPU reference. + pub fn cpu(&self, cpu: CpuId) -> &HostCpu { + &self.cpus[cpu.0] + } + + /// Returns a mutable CPU reference. + pub fn cpu_mut(&mut self, cpu: CpuId) -> &mut HostCpu { + &mut self.cpus[cpu.0] + } + + /// Creates a task assigned to `cpu`. + pub fn create_task(&self, cpu: CpuId) -> TaskRef { + Arc::new(TaskCore::new( + TaskId(self.next_task_id.fetch_add(1, Ordering::AcqRel)), + cpu, + )) + } + + /// Registers a virtual IRQ line. + pub fn register_irq( + &mut self, + mode: IrqMode, + affinity: CpuMask, + callback: impl FnMut(&mut HostSmpRuntime, CpuId, IrqId) + Send + 'static, + ) -> IrqId { + let id = IrqId(self.irqs.len()); + self.irqs.push(IrqLine { + enabled: true, + pending: false, + mode, + affinity, + callback: Some(Box::new(callback)), + }); + id + } + + /// Enables or disables a virtual IRQ line. + pub fn set_irq_enabled(&mut self, irq: IrqId, enabled: bool) { + self.irqs[irq.0].enabled = enabled; + } + + /// Changes virtual IRQ affinity. + pub fn set_irq_affinity(&mut self, irq: IrqId, affinity: CpuMask) { + self.irqs[irq.0].affinity = affinity; + } + + /// Raises a virtual IRQ line. + pub fn raise_irq(&mut self, irq: IrqId) { + self.irqs[irq.0].pending = true; + } + + /// Acknowledges a virtual IRQ line. + pub fn ack_irq(&mut self, irq: IrqId) { + self.irqs[irq.0].pending = false; + } + + /// Delivers one pending IRQ to `cpu`. + pub fn deliver_irq(&mut self, cpu: CpuId) -> bool { + let Some(irq) = self.next_deliverable_irq(cpu) else { + return false; + }; + let mode = self.irqs[irq.0].mode; + if mode == IrqMode::Edge { + self.irqs[irq.0].pending = false; + } + let mut callback = self.irqs[irq.0] + .callback + .take() + .expect("registered IRQ callback missing"); + self.cpus[cpu.0].regs.in_irq = true; + callback(self, cpu, irq); + self.cpus[cpu.0].regs.in_irq = false; + self.cpus[cpu.0].delivered_irqs += 1; + self.irqs[irq.0].callback = Some(callback); + self.irq_epilogue(cpu); + true + } + + /// Delivers at most one pending IRQ per CPU. + pub fn deliver_all_irqs(&mut self) -> usize { + let mut delivered = 0; + for cpu in 0..self.cpus.len() { + if self.deliver_irq(CpuId(cpu)) { + delivered += 1; + } + } + delivered + } + + fn next_deliverable_irq(&self, cpu: CpuId) -> Option { + if !self.cpus[cpu.0].regs.irq_enabled { + return None; + } + self.irqs + .iter() + .enumerate() + .find(|(_, line)| line.enabled && line.pending && line.affinity.contains(cpu)) + .map(|(id, _)| IrqId(id)) + } + + /// Sends a virtual IPI event to a CPU. + pub fn send_ipi_reschedule(&mut self, cpu: CpuId) { + if self.cpus[cpu.0].core.request_ipi(IpiEvent::Reschedule) { + self.raise_irq(self.ipi_irq); + } + } + + /// Sends a virtual IRQ-wake IPI event to a CPU. + pub fn send_ipi_irq_wake(&mut self, cpu: CpuId) { + if self.cpus[cpu.0].core.request_ipi(IpiEvent::IrqWakeDrain) { + self.raise_irq(self.ipi_irq); + } + } + + fn drain_ipi_queue(&mut self, cpu: CpuId) { + let events = self.task_runtime.on_ipi_irq(cpu); + self.cpus[cpu.0].delivered_ipis += events.count(); + if events.contains(IpiEvent::Reschedule) { + self.cpus[cpu.0].regs.need_resched = true; + } + } + + /// Publishes a hard IRQ wake and inserts the task into its target CPU queue. + pub fn wake_from_irq(&mut self, waker: &HardIrqWaker, bits: u64) -> WakeResult { + let result = self.task_runtime.wake_from_irq(waker, bits); + if result.woke() { + self.raise_irq(self.ipi_irq); + } + result + } + + /// Drains one CPU's hard IRQ wake queue. + pub fn drain_irq_wake_queue(&mut self, cpu: CpuId) -> usize { + self.task_runtime.drain_irq_wake_queue(cpu) + } + + /// Runs the virtual IRQ epilogue on `cpu`. + pub fn irq_epilogue(&mut self, cpu: CpuId) { + self.drain_irq_wake_queue(cpu); + } + + /// Schedules a virtual timer that wakes `task`. + pub fn schedule_timer(&mut self, cpu: CpuId, deadline: u64, task: TaskRef) -> u64 { + let id = self.next_timer_id; + self.next_timer_id += 1; + self.timers.push(TimerEvent { + id, + cpu, + deadline, + task, + active: true, + }); + id + } + + /// Cancels a virtual timer. + pub fn cancel_timer(&mut self, id: u64) { + if let Some(timer) = self.timers.iter_mut().find(|timer| timer.id == id) { + timer.active = false; + } + } + + /// Advances virtual monotonic time. + pub fn advance_time(&mut self, nanos: u64) { + self.now_nanos = self.now_nanos.saturating_add(nanos); + let due_cpus: Vec = self + .timers + .iter() + .filter(|timer| timer.active && timer.deadline <= self.now_nanos) + .map(|timer| timer.cpu) + .collect(); + for cpu in due_cpus { + self.raise_irq(self.timer_irq); + self.set_irq_affinity(self.timer_irq, CpuMask::one(cpu)); + } + } + + fn drain_expired_timers(&mut self, cpu: CpuId) { + let mut expired = Vec::new(); + for timer in &mut self.timers { + if timer.active && timer.cpu == cpu && timer.deadline <= self.now_nanos { + timer.active = false; + expired.push(timer.task.clone()); + } + } + for task in expired { + task.set_state(TaskState::Ready); + self.task_runtime.add_ready_task(cpu, task); + self.cpus[cpu.0].regs.need_resched = true; + } + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::Arc; + + use super::*; + use crate::{TaskWaker, WakeSeq}; + + #[test] + fn hard_irq_callback_defers_scheduler_work_until_epilogue() { + let observed_blocked = Arc::new(AtomicBool::new(false)); + let mut runtime = HostSmpRuntime::new(1); + let task = runtime.create_task(CpuId(0)); + task.set_state(TaskState::Blocked); + let waker = TaskWaker::new(task.clone()).to_hard_irq_waker(); + let observed = observed_blocked.clone(); + + let irq = runtime.register_irq(IrqMode::Edge, CpuMask::one(CpuId(0)), move |rt, _, _| { + observed.store(task.state() == TaskState::Blocked, Ordering::Release); + rt.wake_from_irq(&waker, 0x1); + }); + runtime.raise_irq(irq); + + assert!(runtime.deliver_irq(CpuId(0))); + assert!(observed_blocked.load(Ordering::Acquire)); + assert_eq!(runtime.cpu(CpuId(0)).run_queue_len(), 1); + } + + #[test] + fn disabled_irq_stays_pending_until_enabled() { + let hits = Arc::new(AtomicUsize::new(0)); + let mut runtime = HostSmpRuntime::new(1); + let hits_for_irq = hits.clone(); + let irq = runtime.register_irq(IrqMode::Edge, CpuMask::one(CpuId(0)), move |_, _, _| { + hits_for_irq.fetch_add(1, Ordering::AcqRel); + }); + + runtime.set_irq_enabled(irq, false); + runtime.raise_irq(irq); + assert!(!runtime.deliver_irq(CpuId(0))); + assert_eq!(hits.load(Ordering::Acquire), 0); + + runtime.set_irq_enabled(irq, true); + assert!(runtime.deliver_irq(CpuId(0))); + assert_eq!(hits.load(Ordering::Acquire), 1); + } + + #[test] + fn edge_irq_coalesces_multiple_raises_before_delivery() { + let hits = Arc::new(AtomicUsize::new(0)); + let mut runtime = HostSmpRuntime::new(1); + let hits_for_irq = hits.clone(); + let irq = runtime.register_irq(IrqMode::Edge, CpuMask::one(CpuId(0)), move |_, _, _| { + hits_for_irq.fetch_add(1, Ordering::AcqRel); + }); + + runtime.raise_irq(irq); + runtime.raise_irq(irq); + assert!(runtime.deliver_irq(CpuId(0))); + assert!(!runtime.deliver_irq(CpuId(0))); + assert_eq!(hits.load(Ordering::Acquire), 1); + } + + #[test] + fn level_irq_remains_pending_without_spinning() { + let hits = Arc::new(AtomicUsize::new(0)); + let mut runtime = HostSmpRuntime::new(1); + let hits_for_irq = hits.clone(); + let irq = runtime.register_irq(IrqMode::Level, CpuMask::one(CpuId(0)), move |_, _, _| { + hits_for_irq.fetch_add(1, Ordering::AcqRel); + }); + + runtime.raise_irq(irq); + assert!(runtime.deliver_irq(CpuId(0))); + assert_eq!(hits.load(Ordering::Acquire), 1); + assert!(runtime.deliver_irq(CpuId(0))); + assert_eq!(hits.load(Ordering::Acquire), 2); + runtime.ack_irq(irq); + assert!(!runtime.deliver_irq(CpuId(0))); + } + + #[test] + fn irq_affinity_controls_delivery_cpu() { + let mut runtime = HostSmpRuntime::new(2); + let irq = runtime.register_irq(IrqMode::Edge, CpuMask::one(CpuId(1)), |_, _, _| {}); + + runtime.raise_irq(irq); + assert!(!runtime.deliver_irq(CpuId(0))); + assert!(runtime.deliver_irq(CpuId(1))); + assert_eq!(runtime.cpu(CpuId(1)).delivered_irqs(), 1); + } + + #[test] + fn ipi_is_delivered_through_virtual_irq() { + let mut runtime = HostSmpRuntime::new(2); + + runtime.send_ipi_reschedule(CpuId(1)); + assert!(!runtime.cpu(CpuId(1)).regs.need_resched); + assert!(runtime.deliver_irq(CpuId(1))); + assert!(runtime.cpu(CpuId(1)).regs.need_resched); + assert_eq!(runtime.cpu(CpuId(1)).delivered_ipis(), 1); + } + + #[test] + fn remote_hard_irq_wake_drains_on_target_cpu() { + let mut runtime = HostSmpRuntime::new(2); + let task = runtime.create_task(CpuId(1)); + task.set_state(TaskState::Blocked); + let waker = TaskWaker::new(task.clone()).to_hard_irq_waker(); + + let result = runtime.wake_from_irq(&waker, 0x55); + + assert!(result.woke()); + assert_eq!(task.state(), TaskState::Blocked); + assert_eq!(runtime.drain_irq_wake_queue(CpuId(1)), 1); + assert_eq!(task.state(), TaskState::Ready); + assert_eq!(waker.take_bits(), 0x55); + } + + #[test] + fn concurrent_wake_metadata_coalesces_once() { + let mut runtime = HostSmpRuntime::new(1); + let task = runtime.create_task(CpuId(0)); + task.set_state(TaskState::Blocked); + let waker = TaskWaker::new(task.clone()).to_hard_irq_waker(); + let initial_seq: WakeSeq = waker.seq(); + + let first = runtime.wake_from_irq(&waker, 0x1); + let second = runtime.wake_from_irq(&waker, 0x2); + + assert!(first.woke()); + assert!(!second.woke()); + assert_eq!(waker.seq(), initial_seq + 2); + assert_eq!(waker.take_bits(), 0x3); + assert_eq!(runtime.drain_irq_wake_queue(CpuId(0)), 1); + } + + #[test] + fn expired_generation_waker_is_ignored() { + let mut runtime = HostSmpRuntime::new(1); + let task = runtime.create_task(CpuId(0)); + task.set_state(TaskState::Blocked); + let waker = TaskWaker::new(task.clone()).to_hard_irq_waker(); + + task.expire_wakers(); + let result = runtime.wake_from_irq(&waker, 0x1); + + assert!(!result.woke()); + assert_eq!(runtime.drain_irq_wake_queue(CpuId(0)), 0); + } + + #[test] + fn virtual_timer_irq_wakes_due_task_and_cancel_skips_stale_timer() { + let mut runtime = HostSmpRuntime::new(1); + let stale = runtime.create_task(CpuId(0)); + let due = runtime.create_task(CpuId(0)); + stale.set_state(TaskState::Blocked); + due.set_state(TaskState::Blocked); + + let stale_timer = runtime.schedule_timer(CpuId(0), 10, stale.clone()); + runtime.schedule_timer(CpuId(0), 10, due.clone()); + runtime.cancel_timer(stale_timer); + runtime.advance_time(10); + assert!(runtime.deliver_irq(CpuId(0))); + + assert_eq!(stale.state(), TaskState::Blocked); + assert_eq!(due.state(), TaskState::Ready); + assert_eq!(runtime.cpu(CpuId(0)).run_queue_len(), 1); + } +} diff --git a/components/bare-task/src/lib.rs b/components/bare-task/src/lib.rs new file mode 100644 index 0000000000..271489d56b --- /dev/null +++ b/components/bare-task/src/lib.rs @@ -0,0 +1,105 @@ +//! OS-independent task scheduling core. +//! +//! `bare-task` owns the task state, wake, wait, timer, and virtual IRQ test +//! protocols. Operating systems provide hardware and context-switch services +//! through the [`BareTaskOs`] trait generated by `trait-ffi`. + +#![cfg_attr(not(feature = "std"), no_std)] + +extern crate alloc; + +use trait_ffi::*; + +pub mod core; +pub mod cpu; +pub mod local; +pub mod os; +pub mod run_queue; +pub mod runtime; +pub mod scheduler; +pub mod sync; +pub mod timer; +pub mod wait_queue; +pub mod wake; + +#[cfg(feature = "host-test")] +pub mod host; + +pub use core::{CpuId, CpuMask, TaskCore, TaskId, TaskRef, TaskState}; + +pub use cpu::{BareCpuCore, BareTaskRuntime, IpiEvent, IpiEvents, ReadyTaskFromCore}; +pub use local::{LocalExecutorCore, LocalSpawnerCore, SpawnLocalError}; +pub use run_queue::{RunQueueCore, ScheduledTask}; +pub use runtime::{ + BlockOnTaskWake, BlockOnThreadWaker, BlockOnWakeState, RuntimeEventCore, RuntimeEventSeq, + RuntimeEventValue, +}; +pub use scheduler::{ + BaseScheduler, CFSTask, CFScheduler, FifoScheduler, FifoTask, RRScheduler, RRTask, TaskOps, + make_ready, +}; +pub use timer::{TimerKey, TimerRuntimeCore}; +pub use wait_queue::WaitQueueCore; +pub use wake::{HardIrqWaker, IrqWakeQueueCore, TaskWaker, WakeBits, WakeResult, WakeSeq}; + +/// Host/OS services required by the bare task core. +/// +/// Implementations are provided by the OS adapter crate. The core deliberately +/// keeps this ABI narrow and uses only plain values or local newtypes. +#[def_extern_trait] +pub trait BareTaskOs { + /// Returns the number of online CPUs. + fn cpu_num() -> usize; + + /// Returns the current CPU id. + fn this_cpu_id() -> CpuId; + + /// Returns the raw current-task pointer stored by the OS. + fn current_task_ptr() -> *const (); + + /// Stores the raw current-task pointer for the current CPU. + /// + /// # Safety + /// + /// `ptr` must be null or point to a task object that stays valid while it + /// is installed as the current task. + unsafe fn set_current_task_ptr(ptr: *const ()); + + /// Disables local IRQs and returns an opaque previous IRQ state. + fn irq_save_and_disable() -> usize; + + /// Restores a previous local IRQ state. + /// + /// # Safety + /// + /// `state` must have been returned by [`BareTaskOs::irq_save_and_disable`] + /// in the same execution context. + unsafe fn irq_restore(state: usize); + + /// Returns whether local IRQs are currently enabled. + fn irqs_enabled() -> bool; + + /// Returns whether execution is currently inside hard IRQ context. + fn in_irq_context() -> bool; + + /// Waits until an interrupt is observed. + fn wait_for_irqs(); + + /// Returns monotonic time in nanoseconds. + fn monotonic_time_nanos() -> u64; + + /// Programs a one-shot timer deadline in monotonic nanoseconds. + fn set_oneshot_timer(deadline_nanos: u64); + + /// Requests that `cpu` re-evaluate scheduling. + fn request_reschedule(cpu: CpuId); + + /// Requests that `cpu` drain its hard-IRQ wake queue. + fn request_irq_wake(cpu: CpuId); + + /// Returns whether `cpu` is ready to receive cross-CPU requests. + fn wait_until_cpu_ready(cpu: CpuId) -> bool; + + /// Notifies the host that a context switch happened. + fn on_sched_switch(prev: usize, next: usize, next_state: u8); +} diff --git a/components/bare-task/src/local.rs b/components/bare-task/src/local.rs new file mode 100644 index 0000000000..63509dc54f --- /dev/null +++ b/components/bare-task/src/local.rs @@ -0,0 +1,246 @@ +//! Single-threaded cooperative future executor core. + +use alloc::{ + boxed::Box, + collections::VecDeque, + rc::Rc, + sync::{Arc, Weak}, + task::Wake, +}; +use core::{ + future::Future, + marker::PhantomData, + pin::Pin, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, + task::{Context, Poll, Waker}, +}; + +use crate::sync::SpinMutex; + +type LocalFuture = Pin + 'static>>; + +struct LocalTask { + future: SpinMutex>, + queued: AtomicBool, + completed: AtomicBool, + executor: Weak, +} + +// SAFETY: `LocalTask` is shared with `Waker`, so wake callbacks may run on +// another CPU. Those callbacks only touch atomics and the executor ready queue. +// The non-Send future is only accessed from `LocalExecutorCore::poll_ready`; +// `LocalExecutorCore` and `LocalSpawnerCore` are deliberately !Send. +unsafe impl Send for LocalTask {} +// SAFETY: same as the `Send` impl: shared access through a waker never polls or +// otherwise touches the non-Send future. +unsafe impl Sync for LocalTask {} + +impl LocalTask { + fn poll(self: &Arc) { + if self.completed.load(Ordering::Acquire) { + return; + } + let waker = Waker::from(self.clone()); + let mut cx = Context::from_waker(&waker); + let completed = { + let mut future_slot = self.future.lock(); + let Some(future) = future_slot.as_mut() else { + return; + }; + match future.as_mut().poll(&mut cx) { + Poll::Ready(()) => { + *future_slot = None; + true + } + Poll::Pending => false, + } + }; + if completed { + self.completed.store(true, Ordering::Release); + if let Some(executor) = self.executor.upgrade() { + executor.task_count.fetch_sub(1, Ordering::AcqRel); + } + } + } + + fn enqueue(self: &Arc) { + if self.completed.load(Ordering::Acquire) { + return; + } + if self.queued.swap(true, Ordering::AcqRel) { + return; + } + if let Some(executor) = self.executor.upgrade() { + executor.ready.lock().push_back(self.clone()); + (executor.pend)(); + } + } +} + +impl Wake for LocalTask { + fn wake(self: Arc) { + self.enqueue(); + } + + fn wake_by_ref(self: &Arc) { + self.enqueue(); + } +} + +struct LocalExecutorInner { + ready: SpinMutex>>, + active: AtomicBool, + task_count: AtomicUsize, + pend: Arc, +} + +/// Single-threaded cooperative future executor core. +#[derive(Clone)] +pub struct LocalExecutorCore { + inner: Arc, + _not_send: PhantomData>, +} + +impl LocalExecutorCore { + /// Creates an empty local executor core. + pub fn new(pend: Arc) -> Self { + Self { + inner: Arc::new(LocalExecutorInner { + ready: SpinMutex::new(VecDeque::new()), + active: AtomicBool::new(false), + task_count: AtomicUsize::new(0), + pend, + }), + _not_send: PhantomData, + } + } + + /// Returns a spawner tied to this executor. + pub fn spawner(&self) -> LocalSpawnerCore { + LocalSpawnerCore { + inner: self.inner.clone(), + _not_send: PhantomData, + } + } + + /// Enters this executor, rejecting reentrant runs. + pub fn enter(&self) { + assert!( + !self.inner.active.swap(true, Ordering::AcqRel), + "local executor cannot be run reentrantly" + ); + } + + /// Leaves this executor. + pub fn leave(&self) { + self.inner.active.store(false, Ordering::Release); + } + + /// Polls all currently ready tasks. + pub fn poll_ready(&self) -> usize { + let mut polled = 0; + while let Some(task) = self.inner.ready.lock().pop_front() { + task.queued.store(false, Ordering::Release); + task.poll(); + polled += 1; + } + polled + } + + /// Returns whether the executor has ready tasks. + pub fn has_ready_tasks(&self) -> bool { + !self.inner.ready.lock().is_empty() + } + + /// Returns whether at least one local task is still alive. + pub fn has_live_tasks(&self) -> bool { + self.inner.task_count.load(Ordering::Acquire) != 0 + } +} + +/// Handle used to spawn futures onto a [`LocalExecutorCore`]. +#[derive(Clone)] +pub struct LocalSpawnerCore { + inner: Arc, + _not_send: PhantomData>, +} + +impl LocalSpawnerCore { + /// Spawns a future on the local executor. + pub fn spawn_local(&self, future: F) -> Result<(), SpawnLocalError> + where + F: Future + 'static, + { + let task = Arc::new(LocalTask { + future: SpinMutex::new(Some(Box::pin(future))), + queued: AtomicBool::new(true), + completed: AtomicBool::new(false), + executor: Arc::downgrade(&self.inner), + }); + self.inner.task_count.fetch_add(1, Ordering::AcqRel); + self.inner.ready.lock().push_back(task); + (self.inner.pend)(); + Ok(()) + } +} + +/// Error returned by [`LocalSpawnerCore::spawn_local`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SpawnLocalError; + +#[cfg(test)] +mod tests { + use alloc::{rc::Rc, sync::Arc}; + use core::{ + cell::Cell, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use super::LocalExecutorCore; + + #[test] + fn local_executor_polls_spawned_future_and_pends_host() { + let pend_count = Arc::new(AtomicUsize::new(0)); + let pend_count_for_closure = pend_count.clone(); + let executor = LocalExecutorCore::new(Arc::new(move || { + pend_count_for_closure.fetch_add(1, Ordering::AcqRel); + })); + let ran = Arc::new(AtomicUsize::new(0)); + let ran_for_future = ran.clone(); + + executor + .spawner() + .spawn_local(async move { + ran_for_future.fetch_add(1, Ordering::AcqRel); + }) + .unwrap(); + + assert_eq!(pend_count.load(Ordering::Acquire), 1); + executor.enter(); + assert_eq!(executor.poll_ready(), 1); + executor.leave(); + assert_eq!(ran.load(Ordering::Acquire), 1); + assert!(!executor.has_live_tasks()); + } + + #[test] + fn local_executor_accepts_non_send_future() { + let executor = LocalExecutorCore::new(Arc::new(|| {})); + let value = Rc::new(Cell::new(0)); + let value_for_future = value.clone(); + + executor + .spawner() + .spawn_local(async move { + value_for_future.set(1); + }) + .unwrap(); + + executor.enter(); + assert_eq!(executor.poll_ready(), 1); + executor.leave(); + + assert_eq!(value.get(), 1); + assert!(!executor.has_live_tasks()); + } +} diff --git a/components/bare-task/src/os.rs b/components/bare-task/src/os.rs new file mode 100644 index 0000000000..f85723dca3 --- /dev/null +++ b/components/bare-task/src/os.rs @@ -0,0 +1,23 @@ +//! Convenience wrappers around the `BareTaskOs` trait-ffi ABI. + +/// CPU convenience wrappers. +pub mod cpu { + pub use crate::bare_task_os::{cpu_num, current_task_ptr, set_current_task_ptr, this_cpu_id}; +} + +/// IRQ convenience wrappers. +pub mod irq { + pub use crate::bare_task_os::{ + in_irq_context, irq_restore, irq_save_and_disable, irqs_enabled, wait_for_irqs, + }; +} + +/// Time convenience wrappers. +pub mod time { + pub use crate::bare_task_os::{monotonic_time_nanos, set_oneshot_timer}; +} + +/// SMP convenience wrappers. +pub mod smp { + pub use crate::bare_task_os::{request_irq_wake, request_reschedule, wait_until_cpu_ready}; +} diff --git a/components/bare-task/src/run_queue.rs b/components/bare-task/src/run_queue.rs new file mode 100644 index 0000000000..a5f20eacd0 --- /dev/null +++ b/components/bare-task/src/run_queue.rs @@ -0,0 +1,164 @@ +//! Run queue state machine core. + +use crate::{BaseScheduler, TaskOps, TaskState}; + +/// Scheduler item capability required by [`RunQueueCore`]. +pub trait ScheduledTask: Clone { + /// Returns task metadata and state owned by the scheduled item. + fn task_ops(&self) -> &dyn TaskOps; +} + +impl ScheduledTask for alloc::sync::Arc> { + fn task_ops(&self) -> &dyn TaskOps { + &***self + } +} + +impl ScheduledTask for alloc::sync::Arc> { + fn task_ops(&self) -> &dyn TaskOps { + &***self + } +} + +impl ScheduledTask for alloc::sync::Arc> { + fn task_ops(&self) -> &dyn TaskOps { + &***self + } +} + +/// OS-independent run queue core. +/// +/// This owns the scheduler object and the common state transition checks. OS +/// adapters still own per-CPU storage, locking, and context switching. +pub struct RunQueueCore +where + S::SchedItem: ScheduledTask, +{ + scheduler: S, +} + +impl RunQueueCore +where + S: BaseScheduler, + S::SchedItem: ScheduledTask, +{ + /// Creates a run queue core with `scheduler`. + pub const fn new(scheduler: S) -> Self { + Self { scheduler } + } + + /// Initializes the scheduler. + pub fn init(&mut self) { + self.scheduler.init(); + } + + /// Adds a ready task to the scheduler. + pub fn add_task(&mut self, task: S::SchedItem) { + self.scheduler.add_task(task); + } + + /// Removes a task from the scheduler. + pub fn remove_task(&mut self, task: &S::SchedItem) -> Option { + self.scheduler.remove_task(task) + } + + /// Picks the next runnable task. + pub fn pick_next_task(&mut self) -> Option { + self.scheduler.pick_next_task() + } + + /// Puts a runnable previous task back into the scheduler. + pub fn put_prev_task(&mut self, task: S::SchedItem, preempt: bool) { + self.scheduler.put_prev_task(task, preempt); + } + + /// Transitions `task` from `current_state` to `Ready`. + pub fn transition_to_ready(&self, task: &S::SchedItem, current_state: TaskState) -> bool { + crate::make_ready(task.task_ops(), current_state) + } + + /// Transitions `task` from `current_state` to `Ready` and enqueues it. + pub fn put_task_with_state( + &mut self, + task: S::SchedItem, + current_state: TaskState, + preempt: bool, + ) -> bool { + if self.transition_to_ready(&task, current_state) { + self.scheduler.put_prev_task(task, preempt); + true + } else { + false + } + } + + /// Advances the current task accounting by one scheduler tick. + pub fn task_tick(&mut self, current: &S::SchedItem) -> bool { + self.scheduler.task_tick(current) + } + + /// Sets scheduler priority/nice value when supported. + pub fn set_priority(&mut self, task: &S::SchedItem, prio: isize) -> bool { + self.scheduler.set_priority(task, prio) + } +} + +#[cfg(test)] +mod tests { + use alloc::{string::String, sync::Arc}; + + use crate::{ + CpuId, FifoScheduler, FifoTask, RunQueueCore, TaskCore, TaskId, TaskOps, TaskState, + }; + + struct TestTask { + core: TaskCore, + } + + impl TestTask { + fn new(id: u64) -> Arc> { + Arc::new(FifoTask::new(Self { + core: TaskCore::new(TaskId(id), CpuId(0)), + })) + } + } + + impl TaskOps for TestTask { + fn core(&self) -> &TaskCore { + &self.core + } + + fn id_name(&self) -> String { + String::from("test") + } + + fn is_idle(&self) -> bool { + false + } + + fn is_init(&self) -> bool { + false + } + } + + #[test] + fn run_queue_core_put_task_with_state_transitions_and_queues() { + let task = TestTask::new(1); + task.core().set_state(TaskState::Blocked); + + let mut queue = RunQueueCore::new(FifoScheduler::new()); + assert!(queue.put_task_with_state(task.clone(), TaskState::Blocked, false)); + assert_eq!(task.core().state(), TaskState::Ready); + assert!(Arc::ptr_eq(&queue.pick_next_task().unwrap(), &task)); + } + + #[test] + fn run_queue_core_ignores_non_matching_state() { + let task = TestTask::new(1); + task.core().set_state(TaskState::Running); + + let mut queue = RunQueueCore::new(FifoScheduler::new()); + assert!(!queue.put_task_with_state(task, TaskState::Blocked, false)); + assert!(queue.pick_next_task().is_none()); + } +} diff --git a/components/bare-task/src/runtime.rs b/components/bare-task/src/runtime.rs new file mode 100644 index 0000000000..7d00931e3b --- /dev/null +++ b/components/bare-task/src/runtime.rs @@ -0,0 +1,391 @@ +//! State-driven runtime event primitives. + +use alloc::{sync::Arc, task::Wake, vec::Vec}; +use core::{ + sync::atomic::{AtomicU64, Ordering}, + task::{Context, Poll, Waker}, +}; + +use crate::sync::SpinMutex; + +/// Lost-wake state for thread-hosted `block_on` loops. +pub struct BlockOnWakeState { + woke: core::sync::atomic::AtomicBool, +} + +impl BlockOnWakeState { + /// Creates a clear wake state. + pub const fn new() -> Self { + Self { + woke: core::sync::atomic::AtomicBool::new(false), + } + } + + /// Marks the host task as woken. + pub fn mark_woke(&self) { + self.woke.store(true, Ordering::Release); + } + + /// Consumes the local wake flag. + pub fn take_woke(&self) -> bool { + self.woke.swap(false, Ordering::AcqRel) + } + + /// Returns whether a `block_on` loop must re-poll before sleeping. + pub fn should_repoll(&self, observed_seq: u64, current_seq: u64) -> bool { + self.take_woke() || current_seq != observed_seq + } +} + +impl Default for BlockOnWakeState { + fn default() -> Self { + Self::new() + } +} + +/// Host task wake capability used by [`BlockOnThreadWaker`]. +pub trait BlockOnTaskWake: Clone + Send + Sync + 'static { + /// Wakes the host task in task context. + fn wake_task(&self); + + /// Returns the host task wake sequence. + fn wake_seq(&self) -> u64; +} + +/// Generic thread-hosted `block_on` waker core. +pub struct BlockOnThreadWaker { + task_wake: W, + wake_state: BlockOnWakeState, +} + +impl BlockOnThreadWaker { + /// Creates a reference-counted waker core. + pub fn new(task_wake: W) -> Arc { + Arc::new(Self { + task_wake, + wake_state: BlockOnWakeState::new(), + }) + } + + /// Builds a Rust [`Waker`] for polling a future. + pub fn waker(self: &Arc) -> Waker { + Waker::from(self.clone()) + } + + /// Returns the current task wake sequence. + pub fn wake_seq(&self) -> u64 { + self.task_wake.wake_seq() + } + + /// Returns whether the host should re-poll before parking. + pub fn should_repoll(&self, observed_seq: u64) -> bool { + self.wake_state.should_repoll(observed_seq, self.wake_seq()) + } +} + +impl Wake for BlockOnThreadWaker { + fn wake(self: Arc) { + self.wake_by_ref(); + } + + fn wake_by_ref(self: &Arc) { + self.wake_state.mark_woke(); + self.task_wake.wake_task(); + } +} + +/// Mutable sequence cursor for [`RuntimeEventCore`]. +#[derive(Debug, Default)] +pub struct RuntimeEventSeq(AtomicU64); + +impl RuntimeEventSeq { + /// Creates a zero sequence token. + pub const fn new() -> Self { + Self(AtomicU64::new(0)) + } + + /// Returns the raw sequence value. + pub fn get(&self) -> u64 { + self.0.load(Ordering::Acquire) + } + + /// Updates the observed sequence. + pub fn update(&self, seq: RuntimeEventValue) { + self.0.store(seq.get(), Ordering::Release); + } +} + +/// Published runtime event sequence value. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct RuntimeEventValue(u64); + +impl RuntimeEventValue { + /// Returns the raw sequence value. + pub const fn get(self) -> u64 { + self.0 + } +} + +#[derive(Default)] +struct RuntimeEventWaiters { + wakers: Vec, +} + +/// Sticky event source for IRQ/deferred runtime code. +/// +/// `RuntimeEventCore` stores readiness as a monotonically increasing sequence +/// plus coalesced event bits. Wakers are only a hint to re-poll; callers must +/// still consume the published state. +pub struct RuntimeEventCore { + seq: AtomicU64, + bits: AtomicU64, + waiters: SpinMutex, +} + +impl RuntimeEventCore { + /// Creates an empty event source. + pub const fn new() -> Self { + Self { + seq: AtomicU64::new(0), + bits: AtomicU64::new(0), + waiters: SpinMutex::new(RuntimeEventWaiters { wakers: Vec::new() }), + } + } + + /// Returns the latest sequence. + pub fn seq(&self) -> RuntimeEventValue { + RuntimeEventValue(self.seq.load(Ordering::Acquire)) + } + + /// Returns whether at least one event has been published. + pub fn has_unseen_events(&self) -> bool { + self.seq.load(Ordering::Acquire) != 0 + } + + /// Returns whether the event sequence has changed from `observed`. + pub fn has_changed(&self, observed: &RuntimeEventSeq) -> bool { + self.seq.load(Ordering::Acquire) != observed.get() + } + + /// Publishes event bits without waking registered ordinary wakers. + pub fn publish_state(&self, bits: u64) -> RuntimeEventValue { + if bits != 0 { + self.bits.fetch_or(bits, Ordering::AcqRel); + } + RuntimeEventValue(self.seq.fetch_add(1, Ordering::AcqRel) + 1) + } + + /// Publishes event bits and wakes ordinary registered wakers. + pub fn publish(&self, bits: u64) -> RuntimeEventValue { + let seq = self.publish_state(bits); + self.wake_waiters(); + seq + } + + /// Takes all coalesced event bits. + pub fn take_bits(&self) -> u64 { + self.bits.swap(0, Ordering::AcqRel) + } + + /// Polls until the event sequence changes from `observed`. + pub fn poll_changed(&self, observed: &RuntimeEventSeq, cx: &mut Context<'_>) -> Poll { + let seq = self.seq.load(Ordering::Acquire); + if seq != observed.get() { + observed.0.store(seq, Ordering::Release); + return Poll::Ready(seq); + } + + self.register_waker(cx.waker()); + + let seq = self.seq.load(Ordering::Acquire); + if seq != observed.get() { + observed.0.store(seq, Ordering::Release); + Poll::Ready(seq) + } else { + Poll::Pending + } + } + + /// Wakes futures registered with [`poll_changed`](Self::poll_changed). + pub fn wake_waiters(&self) { + let waiters = { + let mut waiters = self.waiters.lock(); + core::mem::take(&mut waiters.wakers) + }; + for waker in waiters { + waker.wake(); + } + } + + fn register_waker(&self, waker: &Waker) { + let mut waiters = self.waiters.lock(); + if let Some(existing) = waiters + .wakers + .iter() + .position(|existing| existing.will_wake(waker)) + { + waiters.wakers[existing] = waker.clone(); + return; + } + waiters.wakers.push(waker.clone()); + } +} + +impl Default for RuntimeEventCore { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use alloc::{sync::Arc, task::Wake}; + use core::{ + sync::atomic::{AtomicUsize, Ordering}, + task::{Context, Poll, Waker}, + }; + + use super::{RuntimeEventCore, RuntimeEventSeq}; + + struct CountWake(AtomicUsize); + + impl Wake for CountWake { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::AcqRel); + } + + fn wake_by_ref(self: &Arc) { + self.0.fetch_add(1, Ordering::AcqRel); + } + } + + #[test] + fn runtime_event_publish_before_poll_is_persistent() { + let event = RuntimeEventCore::new(); + let observed = RuntimeEventSeq::new(); + let seq = event.publish_state(0b101); + + assert_eq!(seq.get(), 1); + assert!(event.has_changed(&observed)); + assert_eq!(event.take_bits(), 0b101); + } + + #[test] + fn runtime_event_register_then_publish_wakes_once() { + let event = RuntimeEventCore::new(); + let observed = RuntimeEventSeq::new(); + let count = Arc::new(CountWake(AtomicUsize::new(0))); + let waker = Waker::from(count.clone()); + let mut cx = Context::from_waker(&waker); + + assert_eq!(event.poll_changed(&observed, &mut cx), Poll::Pending); + event.publish(0b1); + + assert_eq!(count.0.load(Ordering::Acquire), 1); + } + + #[test] + fn block_on_wake_state_repolls_on_local_wake_or_seq_change() { + let state = super::BlockOnWakeState::new(); + + assert!(!state.should_repoll(1, 1)); + state.mark_woke(); + assert!(state.should_repoll(1, 1)); + assert!(state.should_repoll(1, 2)); + } + + #[test] + fn block_on_thread_waker_marks_local_wake_and_calls_task_wake() { + #[derive(Clone)] + struct HostWake(Arc); + + impl crate::BlockOnTaskWake for HostWake { + fn wake_task(&self) { + self.0.fetch_add(1, Ordering::AcqRel); + } + + fn wake_seq(&self) -> u64 { + self.0.load(Ordering::Acquire) as u64 + } + } + + let wake_count = Arc::new(AtomicUsize::new(0)); + let thread_waker = crate::BlockOnThreadWaker::new(HostWake(wake_count.clone())); + let observed = thread_waker.wake_seq(); + let waker = thread_waker.waker(); + + waker.wake_by_ref(); + + assert_eq!(wake_count.load(Ordering::Acquire), 1); + assert!(thread_waker.should_repoll(observed)); + } + + #[test] + fn bare_cpu_core_coalesces_ipi_pending_bits() { + let cpu = crate::BareCpuCore::>::new( + crate::CpuId(0), + crate::FifoScheduler::new(), + ); + + assert!(cpu.request_ipi(crate::IpiEvent::Reschedule)); + assert!(!cpu.request_ipi(crate::IpiEvent::Reschedule)); + assert!(cpu.request_ipi(crate::IpiEvent::IrqWakeDrain)); + + let events = cpu.take_pending_ipis(); + assert_eq!( + events, + crate::IpiEvents::from_events(&[ + crate::IpiEvent::Reschedule, + crate::IpiEvent::IrqWakeDrain, + ]) + ); + assert!(cpu.take_pending_ipis().is_empty()); + } + + #[test] + fn bare_runtime_hard_irq_wake_queues_task_until_epilogue() { + let runtime = crate::BareTaskRuntime::>::new(); + let cpu = runtime.add_cpu(crate::CpuId(0), crate::FifoScheduler::new()); + let task = TestTask::new_core(1, crate::CpuId(0)); + task.set_state(crate::TaskState::Blocked); + let waker = crate::TaskWaker::new(task.clone()).to_hard_irq_waker(); + + let wake = runtime.wake_from_irq(&waker, 0b11); + + assert!(wake.woke()); + assert_eq!(task.state(), crate::TaskState::Blocked); + assert_eq!(cpu.run_queue_len(), 0); + assert_eq!(runtime.drain_irq_wake_queue(crate::CpuId(0)), 1); + assert_eq!(task.state(), crate::TaskState::Ready); + assert_eq!(cpu.run_queue_len(), 1); + assert_eq!(waker.take_bits(), 0b11); + } + + #[test] + fn bare_runtime_timer_irq_marks_service_and_drains_in_task_context() { + let runtime = crate::BareTaskRuntime::>::new(); + let cpu = runtime.add_cpu(crate::CpuId(0), crate::FifoScheduler::new()); + let task = TestTask::new_core(1, crate::CpuId(0)); + task.set_state(crate::TaskState::Blocked); + + runtime.add_task_timer(crate::CpuId(0), 10, task.clone()); + assert!(!cpu.timer_service_pending()); + + runtime.on_timer_irq(crate::CpuId(0), 10); + + assert!(cpu.timer_service_pending()); + assert_eq!(task.state(), crate::TaskState::Blocked); + assert_eq!(runtime.drain_timer_service(crate::CpuId(0), 10), 1); + assert!(!cpu.timer_service_pending()); + assert_eq!(task.state(), crate::TaskState::Ready); + assert_eq!(cpu.run_queue_len(), 1); + } + + struct TestTask; + + impl TestTask { + fn new_core(id: u64, cpu: crate::CpuId) -> crate::TaskRef { + Arc::new(crate::TaskCore::new(crate::TaskId(id), cpu)) + } + } +} diff --git a/components/bare-task/src/scheduler.rs b/components/bare-task/src/scheduler.rs new file mode 100644 index 0000000000..7e98834fff --- /dev/null +++ b/components/bare-task/src/scheduler.rs @@ -0,0 +1,497 @@ +//! Scheduler traits and scheduler cores. + +use alloc::{ + collections::{BTreeMap, VecDeque}, + string::String, + sync::Arc, +}; +use core::{ + ops::Deref, + sync::atomic::{AtomicIsize, Ordering}, +}; + +use crate::{TaskCore, TaskState}; + +/// OS task wrapper capability required by scheduler cores. +pub trait TaskOps { + /// Returns this task's core scheduling state. + fn core(&self) -> &TaskCore; + + /// Returns a human-readable task id/name pair for diagnostics. + fn id_name(&self) -> String; + + /// Returns whether this is the per-CPU idle task. + fn is_idle(&self) -> bool; + + /// Returns whether this is the initial task. + fn is_init(&self) -> bool; +} + +/// Scheduler interface used by the bare task runtime. +/// +/// All tasks in the scheduler are runnable. If a task blocks or exits it must +/// be removed from the scheduler before the state transition becomes visible. +pub trait BaseScheduler { + /// Scheduled entity type. + type SchedItem; + + /// Initializes the scheduler. + fn init(&mut self); + + /// Adds a task to the scheduler. + fn add_task(&mut self, task: Self::SchedItem); + + /// Removes a task by identity and returns it when present. + fn remove_task(&mut self, task: &Self::SchedItem) -> Option; + + /// Picks the next runnable task. + fn pick_next_task(&mut self) -> Option; + + /// Puts the previous task back after a state transition. + fn put_prev_task(&mut self, task: Self::SchedItem, preempt: bool); + + /// Advances scheduler state at each timer tick. + fn task_tick(&mut self, current: &Self::SchedItem) -> bool; + + /// Sets scheduler priority/nice value when supported. + fn set_priority(&mut self, task: &Self::SchedItem, prio: isize) -> bool; +} + +/// FIFO scheduled task wrapper. +pub struct FifoTask(T); + +impl FifoTask { + /// Creates a FIFO task wrapper. + pub const fn new(inner: T) -> Self { + Self(inner) + } + + /// Returns the wrapped task. + pub const fn inner(&self) -> &T { + &self.0 + } +} + +impl Deref for FifoTask { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// FIFO scheduler core. +pub struct FifoScheduler { + ready: VecDeque>>, +} + +impl FifoScheduler { + /// Creates an empty FIFO scheduler. + pub const fn new() -> Self { + Self { + ready: VecDeque::new(), + } + } + + /// Returns scheduler name. + pub fn scheduler_name() -> &'static str { + "FIFO" + } +} + +impl BaseScheduler for FifoScheduler { + type SchedItem = Arc>; + + fn init(&mut self) {} + + fn add_task(&mut self, task: Self::SchedItem) { + self.ready.push_back(task); + } + + fn remove_task(&mut self, task: &Self::SchedItem) -> Option { + let index = self + .ready + .iter() + .position(|ready| Arc::ptr_eq(ready, task))?; + self.ready.remove(index) + } + + fn pick_next_task(&mut self) -> Option { + self.ready.pop_front() + } + + fn put_prev_task(&mut self, task: Self::SchedItem, _preempt: bool) { + self.ready.push_back(task); + } + + fn task_tick(&mut self, _current: &Self::SchedItem) -> bool { + false + } + + fn set_priority(&mut self, _task: &Self::SchedItem, _prio: isize) -> bool { + false + } +} + +impl Default for FifoScheduler { + fn default() -> Self { + Self::new() + } +} + +/// Round-robin scheduled task wrapper. +pub struct RRTask { + inner: T, + time_slice: AtomicIsize, +} + +impl RRTask { + /// Creates a RR task wrapper. + pub const fn new(inner: T) -> Self { + Self { + inner, + time_slice: AtomicIsize::new(S as isize), + } + } + + /// Returns the wrapped task. + pub const fn inner(&self) -> &T { + &self.inner + } + + fn time_slice(&self) -> isize { + self.time_slice.load(Ordering::Acquire) + } + + fn reset_time_slice(&self) { + self.time_slice.store(S as isize, Ordering::Release); + } +} + +impl Deref for RRTask { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +/// Round-robin scheduler core. +pub struct RRScheduler { + ready: VecDeque>>, +} + +impl RRScheduler { + /// Creates an empty RR scheduler. + pub const fn new() -> Self { + Self { + ready: VecDeque::new(), + } + } + + /// Returns scheduler name. + pub fn scheduler_name() -> &'static str { + "Round-robin" + } +} + +impl BaseScheduler for RRScheduler { + type SchedItem = Arc>; + + fn init(&mut self) {} + + fn add_task(&mut self, task: Self::SchedItem) { + self.ready.push_back(task); + } + + fn remove_task(&mut self, task: &Self::SchedItem) -> Option { + let index = self + .ready + .iter() + .position(|ready| Arc::ptr_eq(ready, task))?; + self.ready.remove(index) + } + + fn pick_next_task(&mut self) -> Option { + self.ready.pop_front() + } + + fn put_prev_task(&mut self, task: Self::SchedItem, preempt: bool) { + if preempt && task.time_slice() > 0 { + self.ready.push_front(task); + } else { + task.reset_time_slice(); + self.ready.push_back(task); + } + } + + fn task_tick(&mut self, current: &Self::SchedItem) -> bool { + let old_slice = current.time_slice.fetch_sub(1, Ordering::Release); + old_slice <= 1 + } + + fn set_priority(&mut self, _task: &Self::SchedItem, _prio: isize) -> bool { + false + } +} + +impl Default for RRScheduler { + fn default() -> Self { + Self::new() + } +} + +/// CFS scheduled task wrapper. +pub struct CFSTask { + inner: T, + init_vruntime: AtomicIsize, + delta: AtomicIsize, + nice: AtomicIsize, + id: AtomicIsize, +} + +const NICE_RANGE_POS: usize = 19; +const NICE_RANGE_NEG: usize = 20; + +const NICE2WEIGHT_POS: [isize; NICE_RANGE_POS + 1] = [ + 1024, 820, 655, 526, 423, 335, 272, 215, 172, 137, 110, 87, 70, 56, 45, 36, 29, 23, 18, 15, +]; +const NICE2WEIGHT_NEG: [isize; NICE_RANGE_NEG + 1] = [ + 1024, 1277, 1586, 1991, 2501, 3121, 3906, 4904, 6100, 7620, 9548, 11916, 14949, 18705, 23254, + 29154, 36291, 46273, 56483, 71755, 88761, +]; + +impl CFSTask { + /// Creates a CFS task wrapper. + pub const fn new(inner: T) -> Self { + Self { + inner, + init_vruntime: AtomicIsize::new(0), + delta: AtomicIsize::new(0), + nice: AtomicIsize::new(0), + id: AtomicIsize::new(0), + } + } + + /// Returns the wrapped task. + pub const fn inner(&self) -> &T { + &self.inner + } + + fn get_weight(&self) -> isize { + let nice = self.nice.load(Ordering::Acquire); + if nice >= 0 { + NICE2WEIGHT_POS[nice as usize] + } else { + NICE2WEIGHT_NEG[(-nice) as usize] + } + } + + fn get_id(&self) -> isize { + self.id.load(Ordering::Acquire) + } + + fn get_vruntime(&self) -> isize { + if self.nice.load(Ordering::Acquire) == 0 { + self.init_vruntime.load(Ordering::Acquire) + self.delta.load(Ordering::Acquire) + } else { + self.init_vruntime.load(Ordering::Acquire) + + self.delta.load(Ordering::Acquire) * 1024 / self.get_weight() + } + } + + fn set_vruntime(&self, vruntime: isize) { + self.init_vruntime.store(vruntime, Ordering::Release); + } + + fn set_priority(&self, nice: isize) { + let current = self.get_vruntime(); + self.init_vruntime.store(current, Ordering::Release); + self.delta.store(0, Ordering::Release); + self.nice.store(nice, Ordering::Release); + } + + fn set_id(&self, id: isize) { + self.id.store(id, Ordering::Release); + } + + fn task_tick(&self) { + self.delta.fetch_add(1, Ordering::Release); + } +} + +impl Deref for CFSTask { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +/// Completely fair scheduler core. +pub struct CFScheduler { + ready: BTreeMap<(isize, isize), Arc>>, + min_vruntime: Option, + id_pool: AtomicIsize, +} + +impl CFScheduler { + /// Creates an empty CFS scheduler. + pub const fn new() -> Self { + Self { + ready: BTreeMap::new(), + min_vruntime: None, + id_pool: AtomicIsize::new(0), + } + } + + /// Returns scheduler name. + pub fn scheduler_name() -> &'static str { + "Completely Fair" + } + + fn refresh_min_vruntime(&mut self) { + self.min_vruntime = self + .ready + .first_key_value() + .map(|((min_vruntime, _), _)| *min_vruntime); + } +} + +impl BaseScheduler for CFScheduler { + type SchedItem = Arc>; + + fn init(&mut self) {} + + fn add_task(&mut self, task: Self::SchedItem) { + let vruntime = self.min_vruntime.unwrap_or(0); + let task_id = self.id_pool.fetch_add(1, Ordering::Release); + task.set_vruntime(vruntime); + task.set_id(task_id); + self.ready.insert((vruntime, task_id), task); + self.refresh_min_vruntime(); + } + + fn remove_task(&mut self, task: &Self::SchedItem) -> Option { + let removed = self.ready.remove(&(task.get_vruntime(), task.get_id())); + self.refresh_min_vruntime(); + removed + } + + fn pick_next_task(&mut self) -> Option { + let task = self.ready.pop_first().map(|(_, task)| task); + self.refresh_min_vruntime(); + task + } + + fn put_prev_task(&mut self, task: Self::SchedItem, _preempt: bool) { + let task_id = self.id_pool.fetch_add(1, Ordering::Release); + task.set_id(task_id); + self.ready.insert((task.get_vruntime(), task_id), task); + self.refresh_min_vruntime(); + } + + fn task_tick(&mut self, current: &Self::SchedItem) -> bool { + current.task_tick(); + self.min_vruntime + .is_some_and(|min| current.get_vruntime() > min) + } + + fn set_priority(&mut self, task: &Self::SchedItem, prio: isize) -> bool { + if (-20..=19).contains(&prio) { + task.set_priority(prio); + true + } else { + false + } + } +} + +impl Default for CFScheduler { + fn default() -> Self { + Self::new() + } +} + +/// Transitions `task` to ready and returns whether it should be queued. +pub fn make_ready(task: &(impl TaskOps + ?Sized), current_state: TaskState) -> bool { + task.core() + .transition_state(current_state, TaskState::Ready) + && !task.is_idle() +} + +#[cfg(test)] +mod tests { + use alloc::{string::String, sync::Arc}; + + use super::{BaseScheduler, FifoScheduler, RRScheduler, TaskOps, make_ready}; + use crate::{CpuId, TaskCore, TaskId, TaskState}; + + struct TestTask { + core: TaskCore, + idle: bool, + } + + impl TestTask { + fn new(id: u64) -> Arc { + Arc::new(Self { + core: TaskCore::new(TaskId(id), CpuId(0)), + idle: false, + }) + } + } + + impl TaskOps for TestTask { + fn core(&self) -> &TaskCore { + &self.core + } + + fn id_name(&self) -> String { + String::from("test") + } + + fn is_idle(&self) -> bool { + self.idle + } + + fn is_init(&self) -> bool { + false + } + } + + #[test] + fn fifo_scheduler_preserves_ready_order() { + let first = super::FifoTask::new(1); + let second = super::FifoTask::new(2); + let first = Arc::new(first); + let second = Arc::new(second); + let mut scheduler = FifoScheduler::new(); + + scheduler.add_task(first.clone()); + scheduler.add_task(second.clone()); + + assert!(Arc::ptr_eq(&scheduler.pick_next_task().unwrap(), &first)); + assert!(Arc::ptr_eq(&scheduler.pick_next_task().unwrap(), &second)); + } + + #[test] + fn rr_preempted_task_keeps_remaining_slice_at_front() { + let first = Arc::new(super::RRTask::<_, 5>::new(1)); + let second = Arc::new(super::RRTask::<_, 5>::new(2)); + let mut scheduler = RRScheduler::<_, 5>::new(); + + scheduler.add_task(second.clone()); + scheduler.put_prev_task(first.clone(), true); + + assert!(Arc::ptr_eq(&scheduler.pick_next_task().unwrap(), &first)); + } + + #[test] + fn make_ready_transitions_blocked_non_idle_task() { + let task = TestTask::new(3); + task.core.set_state(TaskState::Blocked); + + assert!(make_ready(task.as_ref(), TaskState::Blocked)); + assert_eq!(task.core.state(), TaskState::Ready); + } +} diff --git a/components/bare-task/src/sync.rs b/components/bare-task/src/sync.rs new file mode 100644 index 0000000000..ea2dac4370 --- /dev/null +++ b/components/bare-task/src/sync.rs @@ -0,0 +1,69 @@ +//! Small synchronization primitives used by the OS-independent task core. + +use core::{ + cell::UnsafeCell, + hint::spin_loop, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicBool, Ordering}, +}; + +/// A minimal non-sleeping spin mutex. +/// +/// This lock does not mask interrupts. OS adapters that require IRQ masking +/// must do that at the adapter boundary before entering bare-task code. +pub struct SpinMutex { + locked: AtomicBool, + value: UnsafeCell, +} + +unsafe impl Send for SpinMutex {} +unsafe impl Sync for SpinMutex {} + +impl SpinMutex { + /// Creates a new spin mutex. + pub const fn new(value: T) -> Self { + Self { + locked: AtomicBool::new(false), + value: UnsafeCell::new(value), + } + } + + /// Locks the mutex. + pub fn lock(&self) -> SpinMutexGuard<'_, T> { + while self + .locked + .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + while self.locked.load(Ordering::Acquire) { + spin_loop(); + } + } + SpinMutexGuard { lock: self } + } +} + +/// Guard returned by [`SpinMutex::lock`]. +pub struct SpinMutexGuard<'a, T> { + lock: &'a SpinMutex, +} + +impl Deref for SpinMutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.lock.value.get() } + } +} + +impl DerefMut for SpinMutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.lock.value.get() } + } +} + +impl Drop for SpinMutexGuard<'_, T> { + fn drop(&mut self) { + self.lock.locked.store(false, Ordering::Release); + } +} diff --git a/components/bare-task/src/timer.rs b/components/bare-task/src/timer.rs new file mode 100644 index 0000000000..60116f585d --- /dev/null +++ b/components/bare-task/src/timer.rs @@ -0,0 +1,133 @@ +//! Timer wheel/runtime core for future timers. + +use alloc::{collections::BTreeMap, vec::Vec}; +use core::task::{Poll, Waker}; + +/// Timer key returned by [`TimerRuntimeCore::add`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct TimerKey { + deadline_nanos: u64, + key: u64, +} + +impl TimerKey { + /// Returns the absolute monotonic deadline in nanoseconds. + pub const fn deadline_nanos(self) -> u64 { + self.deadline_nanos + } +} + +/// Future timer runtime without OS timer programming. +pub struct TimerRuntimeCore { + key: u64, + wheel: BTreeMap, +} + +impl TimerRuntimeCore { + /// Creates an empty timer runtime. + pub const fn new() -> Self { + Self { + key: 0, + wheel: BTreeMap::new(), + } + } + + /// Adds a timer if `deadline_nanos` is in the future. + pub fn add(&mut self, deadline_nanos: u64, now_nanos: u64) -> Option { + if deadline_nanos <= now_nanos { + return None; + } + + let key = TimerKey { + deadline_nanos, + key: self.key, + }; + self.wheel.insert(key, Waker::noop().clone()); + self.key += 1; + Some(key) + } + + /// Polls a timer key. + pub fn poll(&mut self, key: &TimerKey, waker: &Waker) -> Poll<()> { + if let Some(slot) = self.wheel.get_mut(key) { + *slot = waker.clone(); + Poll::Pending + } else { + Poll::Ready(()) + } + } + + /// Cancels a timer key. + pub fn cancel(&mut self, key: &TimerKey) { + self.wheel.remove(key); + } + + /// Returns the next deadline in monotonic nanoseconds. + pub fn next_deadline_nanos(&self) -> Option { + self.wheel.keys().next().map(|key| key.deadline_nanos) + } + + /// Takes all expired timer wakers up to `now_nanos`. + pub fn take_expired(&mut self, now_nanos: u64) -> Vec { + if self.wheel.is_empty() { + return Vec::new(); + } + + let pending = self.wheel.split_off(&TimerKey { + deadline_nanos: now_nanos, + key: u64::MAX, + }); + + let expired = core::mem::replace(&mut self.wheel, pending); + expired.into_values().collect() + } +} + +impl Default for TimerRuntimeCore { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use alloc::{sync::Arc, task::Wake}; + use core::{ + sync::atomic::{AtomicUsize, Ordering}, + task::Waker, + }; + + use super::TimerRuntimeCore; + + struct CountWake(AtomicUsize); + + impl Wake for CountWake { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::AcqRel); + } + + fn wake_by_ref(self: &Arc) { + self.0.fetch_add(1, Ordering::AcqRel); + } + } + + #[test] + fn timer_runtime_orders_cancels_and_takes_expired() { + let mut runtime = TimerRuntimeCore::new(); + let count = Arc::new(CountWake(AtomicUsize::new(0))); + let waker = Waker::from(count.clone()); + let first = runtime.add(10, 0).unwrap(); + let second = runtime.add(20, 0).unwrap(); + + assert_eq!(runtime.next_deadline_nanos(), Some(10)); + assert!(runtime.poll(&first, &waker).is_pending()); + runtime.cancel(&second); + + for waker in runtime.take_expired(10) { + waker.wake(); + } + + assert_eq!(count.0.load(Ordering::Acquire), 1); + assert_eq!(runtime.next_deadline_nanos(), None); + } +} diff --git a/components/bare-task/src/wait_queue.rs b/components/bare-task/src/wait_queue.rs new file mode 100644 index 0000000000..65766ce0a4 --- /dev/null +++ b/components/bare-task/src/wait_queue.rs @@ -0,0 +1,73 @@ +//! Wait queue storage core. + +use alloc::collections::VecDeque; + +/// FIFO wait queue storage. +/// +/// This type intentionally does not block, wake, or change task state by +/// itself. OS adapters hold the appropriate scheduler/wait locks and call into +/// this storage core for queue ordering. +pub struct WaitQueueCore { + queue: VecDeque, +} + +impl WaitQueueCore { + /// Creates an empty wait queue core. + pub const fn new() -> Self { + Self { + queue: VecDeque::new(), + } + } + + /// Pushes one waiter at the back. + pub fn push_back(&mut self, waiter: T) { + self.queue.push_back(waiter); + } + + /// Pops one waiter from the front. + pub fn pop_front(&mut self) -> Option { + self.queue.pop_front() + } + + /// Retains waiters matching `keep`. + pub fn retain(&mut self, mut keep: impl FnMut(&T) -> bool) { + self.queue.retain(|waiter| keep(waiter)); + } + + /// Returns whether the queue is empty. + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } + + /// Returns the number of queued waiters. + pub fn len(&self) -> usize { + self.queue.len() + } +} + +impl Default for WaitQueueCore { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::WaitQueueCore; + + #[test] + fn wait_queue_core_preserves_fifo_and_retain() { + let mut queue = WaitQueueCore::new(); + queue.push_back(1); + queue.push_back(2); + queue.push_back(3); + + queue.retain(|value| *value != 2); + + assert_eq!(queue.len(), 2); + assert_eq!(queue.pop_front(), Some(1)); + assert_eq!(queue.pop_front(), Some(3)); + assert_eq!(queue.pop_front(), None); + assert!(queue.is_empty()); + } +} diff --git a/components/bare-task/src/wake.rs b/components/bare-task/src/wake.rs new file mode 100644 index 0000000000..38fb1c7377 --- /dev/null +++ b/components/bare-task/src/wake.rs @@ -0,0 +1,264 @@ +//! Task wake handles. + +use alloc::sync::{Arc, Weak}; +use core::sync::atomic::{AtomicPtr, Ordering}; + +use crate::{TaskCore, TaskId, TaskRef, TaskState}; + +/// Coalesced wake bits. +pub type WakeBits = u64; + +/// Monotonic wake sequence. +pub type WakeSeq = u64; + +/// Result returned by wake operations. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct WakeResult { + woke: bool, + local: bool, + remote: bool, +} + +impl WakeResult { + /// Creates a result. + pub const fn new(woke: bool, local: bool, remote: bool) -> Self { + Self { + woke, + local, + remote, + } + } + + /// Returns whether a new wake was produced. + pub const fn woke(self) -> bool { + self.woke + } + + /// Returns whether local reschedule/drain work is needed. + pub const fn should_resched(self) -> bool { + self.woke + } + + /// Returns whether the wake targets the current CPU. + pub const fn local(self) -> bool { + self.local + } + + /// Returns whether the wake targets a remote CPU. + pub const fn remote(self) -> bool { + self.remote + } +} + +#[derive(Clone)] +struct WakeHandle { + task: Weak, + task_id: TaskId, + generation: u64, +} + +impl WakeHandle { + fn new(task: TaskRef) -> Self { + Self { + task_id: task.id(), + generation: task.wake_generation(), + task: Arc::downgrade(&task), + } + } + + fn valid_task(&self) -> Option { + let task = self.task.upgrade()?; + if task.id() != self.task_id || !task.wake_generation_matches(self.generation) { + return None; + } + Some(task) + } +} + +/// Cloneable task-context wake handle. +#[derive(Clone)] +pub struct TaskWaker { + handle: WakeHandle, +} + +impl TaskWaker { + /// Creates a task-context waker. + pub fn new(task: TaskRef) -> Self { + Self { + handle: WakeHandle::new(task), + } + } + + /// Creates a hard-IRQ-safe waker for the same task. + pub fn to_hard_irq_waker(&self) -> HardIrqWaker { + HardIrqWaker { + handle: self.handle.clone(), + } + } + + /// Wakes the task from task context. + pub fn wake(&self, bits: WakeBits) -> WakeResult { + let Some(task) = self.handle.valid_task() else { + return WakeResult::default(); + }; + task.publish_wake_bits(bits); + task.bump_wake_seq(); + let woke = matches!(task.state(), TaskState::Blocked | TaskState::Ready); + if task.state() == TaskState::Blocked { + task.set_state(TaskState::Ready); + } + WakeResult::new(woke, true, false) + } + + /// Takes coalesced wake bits. + pub fn take_bits(&self) -> WakeBits { + self.handle + .task + .upgrade() + .map_or(0, |task| task.take_wake_bits()) + } + + /// Returns the current wake sequence. + pub fn seq(&self) -> WakeSeq { + self.handle.task.upgrade().map_or(0, |task| task.wake_seq()) + } +} + +/// Cloneable hard-IRQ-safe wake handle. +#[derive(Clone)] +pub struct HardIrqWaker { + handle: WakeHandle, +} + +impl HardIrqWaker { + /// Publishes wake state from hard IRQ context. + /// + /// This method only touches task-local atomics. Queue insertion and remote + /// CPU notification are performed by the runtime/OS adapter. + pub fn wake_from_irq(&self, bits: WakeBits) -> (Option, WakeResult) { + let Some(task) = self.handle.valid_task() else { + return (None, WakeResult::default()); + }; + task.publish_wake_bits(bits); + task.bump_wake_seq(); + let woke = task.mark_wake_pending(); + (Some(task), WakeResult::new(woke, false, false)) + } + + /// Takes coalesced wake bits. + pub fn take_bits(&self) -> WakeBits { + self.handle + .task + .upgrade() + .map_or(0, |task| task.take_wake_bits()) + } + + /// Returns the current wake sequence. + pub fn seq(&self) -> WakeSeq { + self.handle.task.upgrade().map_or(0, |task| task.wake_seq()) + } +} + +/// Lock-free intrusive MPSC stack used by hard-IRQ wake queues. +pub struct IrqWakeQueueCore { + head: AtomicPtr<()>, +} + +impl IrqWakeQueueCore { + /// Creates an empty IRQ wake queue. + pub const fn new() -> Self { + Self { + head: AtomicPtr::new(core::ptr::null_mut()), + } + } + + /// Pushes `node` and stores the previous head through `set_next`. + /// + /// `node` must remain valid until a successful [`pop`](Self::pop) returns + /// it to the caller. + pub fn push(&self, node: *mut (), mut set_next: impl FnMut(*mut ())) { + let mut head = self.head.load(Ordering::Acquire); + loop { + set_next(head); + match self + .head + .compare_exchange_weak(head, node, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return, + Err(next) => head = next, + } + } + } + + /// Pops one node, using `next_of` to read its intrusive next pointer. + pub fn pop(&self, mut next_of: impl FnMut(*mut ()) -> *mut ()) -> Option<*mut ()> { + loop { + let head = self.head.load(Ordering::Acquire); + if head.is_null() { + return None; + } + let next = next_of(head); + if self + .head + .compare_exchange(head, next, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return Some(head); + } + } + } + + /// Returns whether the queue is currently empty. + pub fn is_empty(&self) -> bool { + self.head.load(Ordering::Acquire).is_null() + } +} + +impl Default for IrqWakeQueueCore { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::{AtomicPtr, Ordering}; + + use super::IrqWakeQueueCore; + + struct Node { + next: AtomicPtr<()>, + } + + #[test] + fn irq_wake_queue_core_pushes_and_pops_intrusive_nodes() { + let queue = IrqWakeQueueCore::new(); + let first = Node { + next: AtomicPtr::new(core::ptr::null_mut()), + }; + let second = Node { + next: AtomicPtr::new(core::ptr::null_mut()), + }; + let first_ptr = (&first as *const Node).cast_mut().cast::<()>(); + let second_ptr = (&second as *const Node).cast_mut().cast::<()>(); + + queue.push(first_ptr, |next| first.next.store(next, Ordering::Release)); + queue.push(second_ptr, |next| { + second.next.store(next, Ordering::Release) + }); + + assert_eq!( + queue.pop(|node| unsafe { &*node.cast::() } + .next + .load(Ordering::Acquire)), + Some(second_ptr) + ); + assert_eq!( + queue.pop(|node| unsafe { &*node.cast::() } + .next + .load(Ordering::Acquire)), + Some(first_ptr) + ); + assert!(queue.is_empty()); + } +} diff --git a/docs/docs/architecture/arceos.md b/docs/docs/architecture/arceos.md index d619bee50a..503239524c 100644 --- a/docs/docs/architecture/arceos.md +++ b/docs/docs/architecture/arceos.md @@ -160,7 +160,6 @@ paging = ["ax-hal/paging", "dep:ax-mm", "dep:axklib"] dma = ["paging"] multitask = ["ax-task/multitask"] smp = ["alloc", "ax-hal/smp", "ax-task?/smp"] -irq = ["ax-hal/irq", "ax-task?/irq", "dep:ax-percpu"] fs = ["ax-driver", "dep:ax-fs"] net = ["ax-driver", "dep:ax-net"] display = ["ax-driver", "dep:ax-display"] diff --git a/docs/docs/components/crates/ax-api.md b/docs/docs/components/crates/ax-api.md index 12c1a7d779..3435226506 100644 --- a/docs/docs/components/crates/ax-api.md +++ b/docs/docs/components/crates/ax-api.md @@ -12,7 +12,7 @@ - 目录角色：ArceOS 公共 API/feature 聚合层 - crate 形态：库 crate - 工作区位置：子工作区 `os/arceos` -- feature 视角：主要通过 `alloc`、`display`、`dma`、`dummy-if-not-enabled`、`fs`、`ipi`、`irq`、`multitask`、`net`、`paging` 控制编译期能力装配。 +- feature 视角：主要通过 `alloc`、`display`、`dma`、`dummy-if-not-enabled`、`fs`、`ipi`、`multitask`、`net`、`paging` 控制编译期能力装配；IRQ 作为基础能力默认存在。 - 关键数据结构：可直接观察到的关键数据结构/对象包括 `AxTimeValue`、`DMAInfo`、`AxTaskHandle`、`AxWaitQueueHandle`。 ### 模块结构 diff --git a/docs/docs/components/crates/ax-hal.md b/docs/docs/components/crates/ax-hal.md index 6d0e5379bb..04f112a48d 100644 --- a/docs/docs/components/crates/ax-hal.md +++ b/docs/docs/components/crates/ax-hal.md @@ -164,12 +164,12 @@ ax-hal = { workspace = true } 更关键的是系统级验证： - ArceOS 最小启动路径，例如 `arceos-helloworld`。 -- 依赖 `paging`、`irq`、`smp` 的场景，如 `ax-mm`、`ax-task` 相关测试。 +- 依赖 `paging`、IRQ、`smp` 的场景，如 `ax-mm`、`ax-task` 相关测试。 - StarryOS 与 Axvisor 的最小 bring-up 路径，验证 HAL 改动没有破坏上层复用。 ### 5.3 覆盖率要求 - `ax-hal` 不以行覆盖率为核心指标，而以“平台组合覆盖”和“启动链覆盖”为核心指标。 -- 涉及 `irq`、`paging`、`tls`、`smp` 的改动，至少应覆盖一条启用 feature 的真实系统路径。 +- 涉及 IRQ、`paging`、`tls`、`smp` 的改动，至少应覆盖一条真实系统路径。 - 涉及链接脚本、内存布局、页表和 trap 的改动，应视为高风险改动，需要跨系统验证。 ## 跨项目定位 diff --git a/docs/docs/components/crates/ax-libc.md b/docs/docs/components/crates/ax-libc.md index 640f6750a6..c931adbce8 100644 --- a/docs/docs/components/crates/ax-libc.md +++ b/docs/docs/components/crates/ax-libc.md @@ -12,7 +12,7 @@ - 目录角色：ArceOS 用户库层 - crate 形态：库 crate - 工作区位置：子工作区 `os/arceos` -- feature 视角：主要通过 `alloc`、`epoll`、`fd`、`fp-simd`、`fs`、`irq`、`multitask`、`net` 等能力 feature 控制编译期能力装配；平台实现固定由 `axplat-dyn` 运行时发现。 +- feature 视角：主要通过 `alloc`、`epoll`、`fd`、`fp-simd`、`fs`、`multitask`、`net` 等能力 feature 控制编译期能力装配；平台实现固定由动态平台路径提供，IRQ 作为基础能力默认存在。 - 关键数据结构：可直接观察到的关键数据结构/对象包括 `MemoryControlBlock`、`CTRL_BLK_SIZE`。 ### 模块结构 diff --git a/docs/docs/components/crates/ax-plat.md b/docs/docs/components/crates/ax-plat.md index 3973f57816..94d46db9bc 100644 --- a/docs/docs/components/crates/ax-plat.md +++ b/docs/docs/components/crates/ax-plat.md @@ -234,7 +234,7 @@ graph TD ```toml [dependencies] -ax-plat = { workspace = true, features = ["irq", "smp"] } +ax-plat = { workspace = true, features = ["smp"] } ``` 构建时由上层选择合适的平台包，例如： @@ -250,14 +250,14 @@ cargo build -p ax-plat --all-features - `ax_percpu::init_primary()` / `init_secondary()` 应尽早调用，因为后续初始化可能依赖当前 CPU ID。 - `MemIf::virt_to_phys()` 只保证对 `phys_to_virt()` 生成的线性映射地址可逆，不能用来翻译任意虚拟地址。 - `busy_wait()` 使用墙钟时间，平台若未正确设置 `epochoffset_nanos()`，墙钟语义可能不准确。 -- `irq` 与 `smp` 是显式 feature，平台包与上层 crate 的 feature 需要一致传播。 +- IRQ 接口默认存在；平台包与上层 crate 只需要继续一致传播 `smp` 等真实可选 feature。 ## 测试 ### 5.1 当前源码中的可验证点 - `mem` 模块已经包含区间重叠检测与差集算法的单元测试，是 `ax-plat` 当前最稳定、最适合主机侧验证的部分。 -- `docs.rs` 配置为 `all-features = true`，意味着文档构建默认覆盖 `irq`/`smp` 等接口可见性。 +- `docs.rs` 配置为 `all-features = true`，意味着文档构建默认覆盖 `smp` 等接口可见性；IRQ 接口默认可见。 - 真实的平台契约验证主要依赖各 `ax-plat-*` 平台包及示例内核的交叉构建与启动冒烟。 ### 5.2 建议的测试分层 @@ -270,7 +270,7 @@ cargo build -p ax-plat --all-features ### 5.3 重点关注的风险 - 接口签名变更会影响所有 `ax-plat-*` 平台包，属于高传播面修改。 -- `irq` 和 `smp` feature 的不一致传播容易造成“接口存在但实现未编译”或“实现存在但上层未启用”的构建问题。 +- `smp` 等 feature 的不一致传播容易造成“接口存在但实现未编译”或“实现存在但上层未启用”的构建问题；IRQ 相关风险主要来自平台实现是否正确返回或处理 `Unsupported`。 - 若平台包错误实现 `phys_to_virt()`/`virt_to_phys()`，上层内存管理和设备映射会出现隐蔽错误。 ## 跨项目定位 diff --git a/docs/docs/components/crates/ax-posix-api.md b/docs/docs/components/crates/ax-posix-api.md index a033367091..4007a8e6d2 100644 --- a/docs/docs/components/crates/ax-posix-api.md +++ b/docs/docs/components/crates/ax-posix-api.md @@ -12,7 +12,7 @@ - 目录角色：ArceOS 公共 API/feature 聚合层 - crate 形态：库 crate - 工作区位置：子工作区 `os/arceos` -- feature 视角：主要通过 `alloc`、`epoll`、`fd`、`fs`、`irq`、`multitask`、`net`、`pipe`、`select`、`smp` 控制编译期能力装配。 +- feature 视角：主要通过 `alloc`、`epoll`、`fd`、`fs`、`multitask`、`net`、`pipe`、`select`、`smp` 控制编译期能力装配；IRQ 作为基础能力默认存在。 - 关键数据结构：该 crate 暴露的数据结构较少，关键复杂度主要体现在模块协作、trait 约束或初始化时序。 ### 模块结构 diff --git a/docs/docs/components/crates/ax-runtime.md b/docs/docs/components/crates/ax-runtime.md index d28966ff26..5e04af392b 100644 --- a/docs/docs/components/crates/ax-runtime.md +++ b/docs/docs/components/crates/ax-runtime.md @@ -12,7 +12,7 @@ - 目录角色：ArceOS 内核模块 - crate 形态：库 crate - 工作区位置：子工作区 `os/arceos` -- feature 视角：主要通过 `alloc`、`ax-driver`、`buddy-slab`、`display`、`dma`、`fs`、`fs-ng`、`input`、`ipi`、`irq` 等（另有 9 个 feature） 控制编译期能力装配。 +- feature 视角：主要通过 `alloc`、`ax-driver`、`buddy-slab`、`display`、`dma`、`fs`、`fs-ng`、`input`、`ipi` 等 feature 控制编译期能力装配；IRQ 初始化与调度闭环默认存在。 - 关键数据结构：可直接观察到的关键数据结构/对象包括 `LogIfImpl`、`AddrTranslatorImpl`、`KlibImpl`、`LOGO`、`INITED_CPUS`、`TRANSLATOR`、`PERIODIC_INTERVAL_NANOS`。 ### 模块结构 diff --git a/docs/docs/components/crates/axklib.md b/docs/docs/components/crates/axklib.md index 25428ec6d7..236b78479a 100644 --- a/docs/docs/components/crates/axklib.md +++ b/docs/docs/components/crates/axklib.md @@ -56,10 +56,10 @@ flowchart TD F --> G["ax-hal::time::busy_wait"] H["axklib::irq::register/set_enable"] --> I["KlibImpl"] - I --> J["ax-hal::irq::* 或 unimplemented!()"] + I --> J["ax-hal::irq::*"] ``` -其中一个很重要的细节是：在 `ax-runtime` 当前实现里，如果没有打开 `irq` feature，`irq_set_enable()` 和 `irq_register()` 会直接 `unimplemented!()`。所以 `axklib` 本身提供的是接口承诺，不保证所有实现方在所有 feature 组合下都完整可用。 +其中一个很重要的细节是：IRQ helper 现在是默认接口。若底层平台没有真实中断控制器，`ax-hal::irq` 实现应返回 `Unsupported` 或 no-op，而不是通过 Cargo feature 隐藏整条调用路径。 ## 核心功能 ### 功能概览 @@ -131,7 +131,7 @@ axklib = { workspace = true } ### 集成测试 - ArceOS 运行时实现能否满足驱动和动态平台代码的需求。 -- 在没有 `irq` feature 的组合下，调用方是否避免误用 IRQ helper。 +- 在平台返回 `Unsupported` 的组合下，调用方是否正确处理 IRQ helper 的失败返回。 ### 覆盖率 - 对 `axklib`，接口兼容性覆盖比单纯行覆盖更重要。 diff --git a/docs/docs/components/crates/axplat-dyn.md b/docs/docs/components/crates/axplat-dyn.md index 7800b4b416..6c2896e6ea 100644 --- a/docs/docs/components/crates/axplat-dyn.md +++ b/docs/docs/components/crates/axplat-dyn.md @@ -28,7 +28,7 @@ | 模块 | 作用 | 关键内容 | | --- | --- | --- | -| `lib.rs` | crate 根与装配层 | 裸机目标限定、模块导入、无 `irq` 时的空中断入口 | +| `lib.rs` | crate 根与装配层 | 裸机目标限定、模块导入、平台入口装配 | | `boot` | 启动 glue | `#[somehal::entry(Kernel)]`、`#[somehal::secondary_entry]`、`Kernel` 的 `MmioOp` 实现 | | `init` | `InitIf` 实现 | trap 初始化、计时器打开、`post_paging()`、后期 IRQ 打开 | | `console` | `ConsoleIf` 实现 | 控制台读写、`\n` 到 `\r\n` 的串口兼容转换 | @@ -167,18 +167,17 @@ flowchart TD - 通过 `build.rs + link.ld` 生成适配当前内核镜像的 `axplat.x` 链接脚本扩展。 - 作为 `ax-hal` 的固定平台实现依赖接入这一动态平台路径。 - 让 `ax-driver` 复用其设备探测与动态块设备封装。 -- 通过 `hv`、`uspace`、`smp`、`irq` feature 把能力向 `somehal` 和 `axplat` 两侧传播。 +- 通过 `hv`、`uspace`、`smp` feature 把能力向 `somehal` 和 `axplat` 两侧传播；IRQ 接口默认存在。 ### 2.2 feature 行为 | Feature | 作用 | | --- | --- | | `smp` | 透传到 `ax-plat/smp`，启用次核入口、次核初始化和 `cpu_boot()` 路径 | -| `irq` | 透传到 `ax-plat/irq`，编译 `irq.rs` 并启用 timer IRQ 相关接口 | | `uspace` | 透传到 `somehal/uspace`，说明该路径允许 `somehal` 切换到含用户态支持的构建 | | `hv` | 透传到 `somehal/hv` 与 `ax-cpu/arm-el2`，为 hypervisor 场景准备 CPU 模式支持 | -需要注意，默认 feature 就是 `["smp", "irq"]`，这意味着该 crate 被设计成优先服务多核且可中断的平台路径，而不是最小单核裸机包。 +需要注意，默认 feature 启用 `smp`，IRQ 接口则作为平台基础能力默认编译；该 crate 被设计成优先服务多核且可中断的平台路径，而不是最小单核裸机包。 ### 2.3 典型使用场景 diff --git a/drivers/ax-driver/Cargo.toml b/drivers/ax-driver/Cargo.toml index a57fac9874..9db689b1e6 100644 --- a/drivers/ax-driver/Cargo.toml +++ b/drivers/ax-driver/Cargo.toml @@ -13,7 +13,6 @@ name = "ax_driver" [features] default = [] -irq = [] pci = ["dep:ax-kspin", "dep:pcie", "dep:rdif-pcie"] block = ["dep:ax-errno", "dep:ax-kspin", "dep:rdif-block"] diff --git a/drivers/ax-driver/src/block/binding.rs b/drivers/ax-driver/src/block/binding.rs index ffabf7b1d1..f963e85bc9 100644 --- a/drivers/ax-driver/src/block/binding.rs +++ b/drivers/ax-driver/src/block/binding.rs @@ -1,13 +1,13 @@ -#[cfg(feature = "irq")] -use alloc::sync::Arc; use alloc::{ boxed::Box, string::{String, ToString}, + sync::Arc, vec::Vec, }; -use core::alloc::Layout; -#[cfg(feature = "irq")] -use core::sync::atomic::{AtomicU64, Ordering}; +use core::{ + alloc::Layout, + sync::atomic::{AtomicU64, Ordering}, +}; use ax_errno::{AxError, AxResult}; use ax_kspin::SpinNoIrq; @@ -31,7 +31,6 @@ pub struct Block { name: String, info: BindingInfo, irq_enabled: bool, - #[cfg(feature = "irq")] irq_handler: Option, interface: Box, queues: SpinNoIrq, @@ -40,7 +39,6 @@ pub struct Block { struct BlockQueues { queue: RuntimeQueue, pool: BlockBufferPool, - #[cfg(feature = "irq")] irq_events: Arc, } @@ -96,13 +94,11 @@ impl BoundDevice for PlatformBlockDevice { } } -#[cfg(feature = "irq")] pub struct BlockIrqHandler { handler: Box, events: Option>, } -#[cfg(feature = "irq")] impl BlockIrqHandler { fn new(handler: Box, events: Arc) -> Self { Self { @@ -127,16 +123,11 @@ impl BlockIrqHandler { } } -#[cfg(not(feature = "irq"))] -pub struct BlockIrqHandler; - -#[cfg(feature = "irq")] #[derive(Default)] struct BlockIrqEvents { queues: AtomicU64, } -#[cfg(feature = "irq")] impl BlockIrqEvents { fn record(&self, event: rdif_block::Event) { let queues = event.queues.bits(); @@ -197,19 +188,12 @@ impl Block { self.info.irq() } - #[cfg(feature = "irq")] pub fn take_irq_handler(&mut self) -> Option<(usize, BlockIrqHandler)> { let irq = self.info.irq_num()?; let handler = self.irq_handler.take()?; Some((irq, handler)) } - #[cfg(not(feature = "irq"))] - pub fn take_irq_handler(&mut self) -> Option<(usize, BlockIrqHandler)> { - let _ = self; - None - } - pub fn num_blocks(&self) -> u64 { self.queues.lock().queue.info().device.num_blocks } @@ -379,7 +363,6 @@ impl RdifBlockDevice { self.interface.disable_irq(); } - #[cfg(feature = "irq")] pub fn take_irq_handler(&mut self, source_id: usize) -> Option<(usize, BlockIrqHandler)> { let irq_num = self.irq_num()?; self.interface @@ -387,18 +370,10 @@ impl RdifBlockDevice { .map(BlockIrqHandler::new_raw) .map(|handler| (irq_num, handler)) } - - #[cfg(not(feature = "irq"))] - pub fn take_irq_handler(&mut self, _source_id: usize) -> Option<(usize, BlockIrqHandler)> { - None - } } impl BlockQueues { - fn new( - queue: RuntimeQueue, - #[cfg(feature = "irq")] irq_events: Arc, - ) -> AxResult { + fn new(queue: RuntimeQueue, irq_events: Arc) -> AxResult { let info = queue.info(); let block_size = info.device.logical_block_size; if block_size == 0 { @@ -418,7 +393,6 @@ impl BlockQueues { size: layout.size(), align: layout.align(), }, - #[cfg(feature = "irq")] irq_events, }) } @@ -449,12 +423,10 @@ impl BlockQueues { .map_err(map_blk_err_to_ax_err)?; match status { RequestStatus::Complete => { - #[cfg(feature = "irq")] let _ = self.irq_events.take_queue(self.queue.id()); return Ok(()); } RequestStatus::Pending => { - #[cfg(feature = "irq")] if self.irq_events.take_queue(self.queue.id()) { continue; } @@ -474,13 +446,11 @@ impl BlockQueues { .map_err(|err| map_blk_err_to_ax_err(err.into()))?; match status { RequestPoll::Ready(completed) => { - #[cfg(feature = "irq")] let _ = self.irq_events.take_queue(self.queue.id()); completed.result.map_err(map_blk_err_to_ax_err)?; return Ok(completed); } RequestPoll::Pending => { - #[cfg(feature = "irq")] if self.irq_events.take_queue(self.queue.id()) { continue; } @@ -512,7 +482,6 @@ impl BlockBufferPool { } impl RuntimeQueue { - #[cfg(feature = "irq")] fn id(&self) -> usize { match self { Self::Legacy(queue) => queue.id(), @@ -560,40 +529,25 @@ impl TryFrom> for Block { .map(RuntimeQueue::Owned) .or_else(|| interface.create_queue().map(RuntimeQueue::Legacy)) .ok_or(AxError::BadState)?; - #[cfg(feature = "irq")] let irq_events = Arc::new(BlockIrqEvents::default()); - let queues = BlockQueues::new( - queue, - #[cfg(feature = "irq")] - Arc::clone(&irq_events), - )?; + let queues = BlockQueues::new(queue, Arc::clone(&irq_events))?; - #[cfg(feature = "irq")] let irq_handler = irq .as_ref() .and_then(|_| take_legacy_irq_handler(interface.as_mut())) .map(|handler| BlockIrqHandler::new(handler, irq_events)); drop(dev); - #[cfg(feature = "irq")] let info = if irq_handler.is_some() { info } else { BindingInfo::empty() }; - #[cfg(feature = "irq")] - let irq_handler = irq_handler; - #[cfg(not(feature = "irq"))] - let info = { - let _ = irq; - BindingInfo::empty() - }; Ok(Self { name, info, irq_enabled: interface.is_irq_enabled(), - #[cfg(feature = "irq")] irq_handler, interface, queues: SpinNoIrq::new(queues), @@ -733,7 +687,6 @@ pub fn take_raw_block_devices() -> Vec { #[deprecated(note = "use RdifBlockDevice")] pub type RawBlockDevice = RdifBlockDevice; -#[cfg(feature = "irq")] fn take_legacy_irq_handler( interface: &mut dyn Interface, ) -> Option> { @@ -1196,7 +1149,6 @@ mod tests { name: String::from("test-block"), info: BindingInfo::empty(), irq_enabled: false, - #[cfg(feature = "irq")] irq_handler: None, interface: Box::new(TestInterface), queues: SpinNoIrq::new(BlockQueues { @@ -1207,7 +1159,6 @@ mod tests { size: layout.size(), align: layout.align(), }, - #[cfg(feature = "irq")] irq_events: Arc::new(BlockIrqEvents::default()), }), } @@ -1361,7 +1312,6 @@ mod tests { size: layout.size(), align: layout.align(), }, - #[cfg(feature = "irq")] irq_events: Arc::new(BlockIrqEvents::default()), }; let mut buf = [0_u8; 4096]; diff --git a/drivers/blk/nvme-driver/src/block.rs b/drivers/blk/nvme-driver/src/block.rs index b3faefa6f0..0266c02877 100644 --- a/drivers/blk/nvme-driver/src/block.rs +++ b/drivers/blk/nvme-driver/src/block.rs @@ -11,7 +11,7 @@ use log::warn; use rdif_block::{ BlkError, CompletionSink, DeviceInfo, DriverGeneric, Event, IQueue, IdList, Interface, IrqHandler, IrqSourceInfo, IrqSourceList, QueueInfo, QueueLimits, Request, RequestFlags, - RequestId, RequestOp, RequestStatus, validate_request, + RequestGeneration, RequestId, RequestOp, RequestStatus, RequestToken, validate_request, }; use crate::{ @@ -286,6 +286,7 @@ struct NvmeQueueCore { queue: UnsafeCell, state: UnsafeCell, completion_cache: CompletionCache, + slot_generations: Vec, state_claimed: AtomicBool, cq_claimed: AtomicBool, } @@ -298,6 +299,7 @@ struct NvmeQueueState { struct RequestSlot { state: SlotState, + generation: RequestGeneration, prp_list: Option>, } @@ -312,6 +314,7 @@ enum SlotState { #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct CachedCompletion { cid: usize, + generation: RequestGeneration, status: CompletionStatus, } @@ -328,6 +331,7 @@ struct CompletionCache { struct CompletionCacheEntry { ready: AtomicBool, + generation: AtomicU64, success: AtomicBool, raw_status: AtomicU16, result: AtomicU64, @@ -355,8 +359,13 @@ impl NvmeQueueCore { let mut slots = Vec::with_capacity(depth + 1); slots.resize_with(depth + 1, || RequestSlot { state: SlotState::Free, + generation: RequestGeneration::default(), prp_list: None, }); + let mut slot_generations = Vec::with_capacity(depth + 1); + slot_generations.resize_with(depth + 1, || { + AtomicU64::new(RequestGeneration::default().get()) + }); let free_cids = (1..=depth).rev().collect(); Arc::new(Self { @@ -374,6 +383,7 @@ impl NvmeQueueCore { free_prp_lists: prp_lists, }), completion_cache: CompletionCache::new(depth + 1), + slot_generations, state_claimed: AtomicBool::new(false), cq_claimed: AtomicBool::new(false), }) @@ -440,13 +450,12 @@ impl NvmeQueueCore { } fn drain_irq_completions(&self) -> bool { - self.try_with_cq_claim(drain_hardware_completions_to_vec) - .map(|completions| self.cache_completions(completions)) + self.try_with_cq_claim(|queue| self.drain_hardware_completions_to_cache(queue)) .unwrap_or(true) } fn drain_completions(&self) -> bool { - let completions = self.with_cq_claim(drain_hardware_completions_to_vec); + let completions = self.with_cq_claim(|queue| self.drain_hardware_completions_to_vec(queue)); self.cache_completions(completions) } @@ -457,6 +466,51 @@ impl NvmeQueueCore { self.completion_cache.extend(completions); true } + + fn publish_slot_generation(&self, cid: usize, generation: RequestGeneration) { + if let Some(slot_generation) = self.slot_generations.get(cid) { + slot_generation.store(generation.get(), Ordering::Release); + } + } + + fn request_token_for(&self, request: RequestId) -> Option { + let cid = usize::from(request); + self.with_claim(|state| { + let slot = state.slots.get(cid)?; + (!matches!(slot.state, SlotState::Free)) + .then_some(RequestToken::new(request, slot.generation)) + }) + } + + fn generation_snapshot_for_cid(&self, cid: usize) -> Option { + self.slot_generations + .get(cid) + .map(|generation| RequestGeneration::new(generation.load(Ordering::Acquire))) + } + + fn drain_hardware_completions_to_cache(&self, queue: &HardwareQueue) -> bool { + let mut completed = false; + while let Some(completion) = queue.poll_completion() { + completed = true; + let cid = usize::from(completion.command_id); + if let Some(generation) = self.generation_snapshot_for_cid(cid) { + self.completion_cache + .record_irq_completion(completion, generation); + } + } + completed + } + + fn drain_hardware_completions_to_vec(&self, queue: &HardwareQueue) -> Vec { + let mut completions = Vec::new(); + while let Some(completion) = queue.poll_completion() { + let cid = usize::from(completion.command_id); + if let Some(generation) = self.generation_snapshot_for_cid(cid) { + completions.push(CachedCompletion::from_completion(completion, generation)); + } + } + completions + } } // SAFETY: Slot, CID, and completion-cache access is serialized through @@ -471,7 +525,11 @@ unsafe impl Sync for NvmeQueueCore {} impl NvmeQueueState { fn alloc_cid(&mut self) -> Result { - self.free_cids.pop().ok_or(BlkError::Retry) + let cid = self.free_cids.pop().ok_or(BlkError::Retry)?; + if let Some(slot) = self.slots.get_mut(cid) { + slot.generation = slot.generation.next(); + } + Ok(cid) } fn free_cid(&mut self, cid: usize) { @@ -568,14 +626,6 @@ impl NvmeQueueState { } } -fn drain_hardware_completions_to_vec(queue: &HardwareQueue) -> Vec { - let mut completions = Vec::new(); - while let Some(completion) = queue.poll_completion() { - completions.push(CachedCompletion::from(completion)); - } - completions -} - impl CompletionCache { fn new(capacity: usize) -> Self { let mut entries = Vec::with_capacity(capacity); @@ -593,6 +643,9 @@ impl CompletionCache { let Some(entry) = self.entries.get(completion.cid) else { return; }; + entry + .generation + .store(completion.generation.get(), Ordering::Relaxed); entry .success .store(completion.status.success, Ordering::Relaxed); @@ -605,6 +658,10 @@ impl CompletionCache { entry.ready.store(true, Ordering::Release); } + fn record_irq_completion(&self, completion: NvmeCompletion, generation: RequestGeneration) { + self.record(CachedCompletion::from_completion(completion, generation)); + } + fn drain_into_slots(&self, queue_id: usize, slots: &mut [RequestSlot]) -> usize { let mut consumed = 0; for (cid, entry) in self.entries.iter().enumerate() { @@ -614,6 +671,13 @@ impl CompletionCache { let Some(slot) = slots.get_mut(cid) else { continue; }; + let generation = RequestGeneration::new(entry.generation.load(Ordering::Relaxed)); + if slot.generation != generation { + continue; + } + if slot.state != SlotState::Pending { + continue; + } let status = CompletionStatus { success: entry.success.load(Ordering::Relaxed), raw_status: entry.raw_status.load(Ordering::Relaxed), @@ -638,6 +702,7 @@ impl CompletionCacheEntry { fn new() -> Self { Self { ready: AtomicBool::new(false), + generation: AtomicU64::new(0), success: AtomicBool::new(false), raw_status: AtomicU16::new(0), result: AtomicU64::new(0), @@ -645,10 +710,11 @@ impl CompletionCacheEntry { } } -impl From for CachedCompletion { - fn from(completion: NvmeCompletion) -> Self { +impl CachedCompletion { + fn from_completion(completion: NvmeCompletion, generation: RequestGeneration) -> Self { Self { cid: usize::from(completion.command_id), + generation, status: CompletionStatus { success: completion.status.is_success(), raw_status: completion.status.0, @@ -682,6 +748,8 @@ unsafe impl IQueue for NvmeBlockQueue { state.consume_cached_completions(queue_id, &self.core.completion_cache); let cid = state.alloc_cid()?; + let generation = state.slots[cid].generation; + self.core.publish_slot_generation(cid, generation); let command = match state.build_command(namespace, page_size, cid, &request) { Ok(command) => command, Err(err) => { @@ -696,6 +764,10 @@ unsafe impl IQueue for NvmeBlockQueue { }) } + fn request_token(&self, request: RequestId) -> Option { + self.core.request_token_for(request) + } + fn poll_request(&mut self, request: RequestId) -> Result { let queue_id = self.core.id(); self.core.drain_completions(); @@ -1006,19 +1078,65 @@ mod tests { assert_eq!(slots[1].state, SlotState::Complete); } + #[test] + fn cached_completion_does_not_complete_free_slot() { + let cache = CompletionCache::new(2); + let mut slots = test_slots(2); + + cache.extend(alloc::vec![CachedCompletion::success(1)]); + + assert_eq!(cache.drain_into_slots(0, &mut slots), 0); + assert_eq!(slots[1].state, SlotState::Free); + } + + #[test] + fn irq_cached_completion_is_visible_to_task_context() { + let cache = CompletionCache::new(2); + let mut slots = test_slots(2); + slots[1].state = SlotState::Pending; + slots[1].generation = rdif_block::RequestGeneration::new(7); + + cache.record_irq_completion( + crate::queue::NvmeCompletion { + command_id: 1, + status: crate::queue::CompletionStatus(1), + ..Default::default() + }, + rdif_block::RequestGeneration::new(7), + ); + + assert_eq!(cache.drain_into_slots(0, &mut slots), 1); + assert_eq!(slots[1].state, SlotState::Complete); + } + fn test_slots(count: usize) -> alloc::vec::Vec { (0..count) .map(|_| RequestSlot { state: SlotState::Free, + generation: Default::default(), prp_list: None, }) .collect() } + #[test] + fn stale_cached_completion_does_not_complete_reused_slot() { + let cache = CompletionCache::new(2); + let mut slots = test_slots(2); + slots[1].state = SlotState::Pending; + slots[1].generation = rdif_block::RequestGeneration::new(7); + + cache.extend(alloc::vec![CachedCompletion::success(1)]); + + assert_eq!(cache.drain_into_slots(0, &mut slots), 0); + assert_eq!(slots[1].state, SlotState::Pending); + } + impl CachedCompletion { const fn success(cid: usize) -> Self { Self { cid, + generation: rdif_block::RequestGeneration::new(1), status: CompletionStatus { success: true, raw_status: 0, @@ -1030,6 +1148,7 @@ mod tests { const fn failed(cid: usize, raw_status: u16) -> Self { Self { cid, + generation: rdif_block::RequestGeneration::new(1), status: CompletionStatus { success: false, raw_status, diff --git a/drivers/interface/rdif-block/src/interface.rs b/drivers/interface/rdif-block/src/interface.rs index 2226f950c1..a3413610a5 100644 --- a/drivers/interface/rdif-block/src/interface.rs +++ b/drivers/interface/rdif-block/src/interface.rs @@ -2,7 +2,8 @@ use alloc::{boxed::Box, vec::Vec}; use crate::{ BlkError, DeviceInfo, DriverGeneric, IrqHandler, IrqSourceList, OwnedRequest, PollError, - QueueInfo, QueueLimits, Request, RequestId, RequestPoll, RequestStatus, SubmitError, + QueueInfo, QueueLimits, Request, RequestId, RequestPoll, RequestStatus, RequestToken, + SubmitError, }; pub type BInterface = Box; @@ -50,6 +51,11 @@ pub trait IQueueOwned: Send + 'static { fn submit_request(&mut self, request: OwnedRequest) -> Result; + fn request_token(&self, request: RequestId) -> Option { + let _ = request; + None + } + fn poll_request(&mut self, request: RequestId) -> Result; fn cancel_request(&mut self, request: RequestId) -> Result; @@ -87,6 +93,13 @@ impl QueueHandle { .submit_request(request) } + pub fn request_token(&self, request: RequestId) -> Option { + self.queue + .as_ref() + .expect("owned queue handle must contain queue") + .request_token(request) + } + pub fn poll_request(&mut self, request: RequestId) -> Result { self.queue .as_mut() @@ -130,6 +143,11 @@ pub unsafe trait IQueue: Send + 'static { fn submit_request(&mut self, request: Request<'_>) -> Result; + fn request_token(&self, request: RequestId) -> Option { + let _ = request; + None + } + fn poll_request(&mut self, request: RequestId) -> Result; /// Poll a set of in-flight requests and report observed terminal results. diff --git a/drivers/interface/rdif-block/src/irq.rs b/drivers/interface/rdif-block/src/irq.rs index c844c9c361..586277f5a5 100644 --- a/drivers/interface/rdif-block/src/irq.rs +++ b/drivers/interface/rdif-block/src/irq.rs @@ -1,6 +1,6 @@ use alloc::vec::Vec; -use crate::RequestId; +use crate::{RequestId, RequestToken}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct IrqSourceInfo { @@ -86,6 +86,10 @@ pub enum CompletionHint { queue_id: usize, request_id: RequestId, }, + Token { + queue_id: usize, + token: RequestToken, + }, Batch { queue_id: usize, ids: CompletionIds, @@ -97,6 +101,7 @@ impl CompletionHint { match self { Self::Queue { queue_id } | Self::Request { queue_id, .. } + | Self::Token { queue_id, .. } | Self::Batch { queue_id, .. } => queue_id, } } @@ -229,6 +234,15 @@ impl Event { } } + pub fn push_token(&mut self, queue_id: usize, token: RequestToken) { + if !self + .completions + .push(CompletionHint::Token { queue_id, token }) + { + self.queues.insert(queue_id); + } + } + pub fn push_hint(&mut self, hint: CompletionHint) { if let CompletionHint::Queue { queue_id } = hint { self.queues.insert(queue_id); @@ -262,9 +276,13 @@ mod tests { let mut event = Event::none(); event.push_queue(3); event.push_request(3, RequestId::new(7)); + event.push_token( + 3, + crate::RequestToken::new(RequestId::new(8), crate::RequestGeneration::new(2)), + ); assert!(event.queues.contains(3)); - assert_eq!(event.completions.len(), 2); + assert_eq!(event.completions.len(), 3); } #[test] diff --git a/drivers/interface/rdif-block/src/lib.rs b/drivers/interface/rdif-block/src/lib.rs index 9ed7d28cc7..bf8c6c3d39 100644 --- a/drivers/interface/rdif-block/src/lib.rs +++ b/drivers/interface/rdif-block/src/lib.rs @@ -26,7 +26,7 @@ pub use planner::{ }; pub use rdif_base::{DriverGeneric, KError, io}; pub use request::{ - Buffer, CompletedRequest, OwnedRequest, PollError, Request, RequestFlags, RequestId, RequestOp, - RequestPoll, RequestStatus, Segment, SubmitError, validate_owned_request, - validate_owned_request_shape, validate_request, validate_request_shape, + Buffer, CompletedRequest, OwnedRequest, PollError, Request, RequestFlags, RequestGeneration, + RequestId, RequestOp, RequestPoll, RequestStatus, RequestToken, Segment, SubmitError, + validate_owned_request, validate_owned_request_shape, validate_request, validate_request_shape, }; diff --git a/drivers/interface/rdif-block/src/request.rs b/drivers/interface/rdif-block/src/request.rs index f7a16a7ae2..0a28df53f4 100644 --- a/drivers/interface/rdif-block/src/request.rs +++ b/drivers/interface/rdif-block/src/request.rs @@ -24,6 +24,47 @@ impl From for usize { } } +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RequestGeneration(u64); + +impl RequestGeneration { + pub const fn new(generation: u64) -> Self { + Self(generation) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn next(self) -> Self { + Self(self.0.wrapping_add(1)) + } +} + +impl Default for RequestGeneration { + fn default() -> Self { + Self(1) + } +} + +/// Stable identity for one submitted request lifetime. +/// +/// `RequestId` may be reused by queue implementations after completion. A +/// token combines the driver-visible ID with a generation so completion caches +/// can reject stale observations from an older lifetime of the same ID. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RequestToken { + pub id: RequestId, + pub generation: RequestGeneration, +} + +impl RequestToken { + pub const fn new(id: RequestId, generation: RequestGeneration) -> Self { + Self { id, generation } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RequestStatus { Pending, diff --git a/drivers/interface/rdif-serial/src/core.rs b/drivers/interface/rdif-serial/src/core.rs index 05b5955b71..b97197548e 100644 --- a/drivers/interface/rdif-serial/src/core.rs +++ b/drivers/interface/rdif-serial/src/core.rs @@ -2,12 +2,12 @@ use alloc::{boxed::Box, sync::Arc}; use core::{ cell::{Cell, UnsafeCell}, marker::PhantomData, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, + sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, }; use crate::{ Config, ConfigError, InterruptMask, IrqSource, RawUart, RxFlag, RxItem, SerialCounters, - SerialIrqOutcome, SpscRing, + SerialEventSnapshot, SerialIrqOutcome, SpscRing, }; pub const DEFAULT_TX_CAP: usize = 4097; @@ -273,6 +273,7 @@ pub struct SerialPort>>, tx: Arc>, rx: Arc>, + events: Arc, counters: Arc, } @@ -281,9 +282,88 @@ pub struct SerialIrqHandler>>, tx: Arc>, rx: Arc>, + events: Arc, counters: Arc, } +struct SerialEventState { + rx_seq: AtomicU64, + tx_seq: AtomicU64, + budget_exhausted: AtomicBool, + error: AtomicBool, + hangup: AtomicBool, + service_needed: AtomicBool, +} + +impl SerialEventState { + fn new() -> Self { + Self { + rx_seq: AtomicU64::new(0), + tx_seq: AtomicU64::new(0), + budget_exhausted: AtomicBool::new(false), + error: AtomicBool::new(false), + hangup: AtomicBool::new(false), + service_needed: AtomicBool::new(false), + } + } + + fn publish_rx_ready(&self, count: usize) { + if count == 0 { + return; + } + self.rx_seq.fetch_add(count as u64, Ordering::AcqRel); + } + + fn publish_tx_space(&self, count: usize) { + if count == 0 { + return; + } + self.tx_seq.fetch_add(count as u64, Ordering::AcqRel); + } + + fn publish_error(&self) { + self.error.store(true, Ordering::Release); + self.rx_seq.fetch_add(1, Ordering::AcqRel); + } + + fn publish_service_needed(&self) { + self.budget_exhausted.store(true, Ordering::Release); + self.service_needed.store(true, Ordering::Release); + self.rx_seq.fetch_add(1, Ordering::AcqRel); + } + + fn clear_service_needed(&self) { + self.budget_exhausted.store(false, Ordering::Release); + self.service_needed.store(false, Ordering::Release); + } + + fn finish_outcome( + &self, + out: &mut SerialIrqOutcome, + tx: &TxState, + rx: &RxState, + ) { + out.snapshot = self.snapshot(tx, rx); + } + + fn snapshot( + &self, + tx: &TxState, + rx: &RxState, + ) -> SerialEventSnapshot { + SerialEventSnapshot { + rx_seq: self.rx_seq.load(Ordering::Acquire), + tx_seq: self.tx_seq.load(Ordering::Acquire), + rx_ready: !rx.ring.is_empty(), + tx_space: tx.write_room() > 0, + budget_exhausted: self.budget_exhausted.load(Ordering::Acquire), + error: self.error.load(Ordering::Acquire), + hangup: self.hangup.load(Ordering::Acquire), + service_needed: self.service_needed.load(Ordering::Acquire), + } + } +} + impl SerialPort { pub fn split(raw: impl RawUart, owner: OwnerId) -> SerialParts { Self::split_boxed(Box::new(raw), owner) @@ -293,12 +373,14 @@ impl SerialPort { let core = Arc::new(OwnerCell::new(CoreInner::new(raw))); let tx = Arc::new(TxState::new()); let rx = Arc::new(RxState::new()); + let events = Arc::new(SerialEventState::new()); let counters = Arc::new(SerialCountersAtomic::new()); let port = Arc::new(Self { owner, core: core.clone(), tx: tx.clone(), rx: rx.clone(), + events: events.clone(), counters: counters.clone(), }); let irq = SerialIrqHandler { @@ -306,6 +388,7 @@ impl SerialPort { core: core.clone(), tx: tx.clone(), rx: rx.clone(), + events, counters, }; SerialParts { @@ -415,6 +498,7 @@ impl SerialIrqHandler { fn handle_locked(&self, core: &mut CoreInner) -> SerialIrqOutcome { let mut out = SerialIrqOutcome::default(); if core.state != PortState::Running { + self.events.finish_outcome(&mut out, &self.tx, &self.rx); return out; } @@ -440,11 +524,13 @@ impl SerialIrqHandler { let service = self.service_rx(core, rx_budget); rx_budget = rx_budget.saturating_sub(service.consumed); out.rx_pushed += service.published; + self.events.publish_rx_ready(service.published); } if snapshot.sources.contains(IrqSource::TX_SPACE) { let sent = self.service_tx(core, tx_budget, &mut out); tx_budget = tx_budget.saturating_sub(sent); + self.events.publish_tx_space(sent); } if snapshot.sources.contains(IrqSource::MODEM_STATUS) { @@ -457,12 +543,14 @@ impl SerialIrqHandler { if rx_budget == 0 || tx_budget == 0 { out.budget_exhausted = true; + self.events.publish_service_needed(); self.counters .irq_budget_exhausted .fetch_add(1, Ordering::Relaxed); break; } } + self.events.finish_outcome(&mut out, &self.tx, &self.rx); out } @@ -477,14 +565,17 @@ impl SerialIrqHandler { match sample.flag { RxFlag::Normal => {} RxFlag::Break => { + self.events.publish_error(); self.counters.rx_breaks.fetch_add(1, Ordering::Relaxed); } RxFlag::Parity => { + self.events.publish_error(); self.counters .rx_parity_errors .fetch_add(1, Ordering::Relaxed); } RxFlag::Framing => { + self.events.publish_error(); self.counters .rx_framing_errors .fetch_add(1, Ordering::Relaxed); @@ -506,6 +597,7 @@ impl SerialIrqHandler { } if sample.overrun { + self.events.publish_error(); self.rx.overrun.fetch_add(1, Ordering::Relaxed); self.counters .rx_fifo_overruns @@ -571,17 +663,33 @@ impl SerialPort { ) -> SerialIrqOutcome { let mut out = SerialIrqOutcome::default(); if core.state != PortState::Running { + self.events.finish_outcome(&mut out, &self.tx, &self.rx); return out; } if work.contains(SerialSoftWork::TX_KICK) { - self.service_tx(core, TX_KICK_BUDGET, &mut out); + let sent = self.service_tx(core, TX_KICK_BUDGET, &mut out); + self.events.publish_tx_space(sent); } if work.contains(SerialSoftWork::RESERVICE) { let rx = self.service_rx(core, RX_IRQ_BUDGET); out.rx_pushed += rx.published; - self.service_tx(core, TX_IRQ_BUDGET, &mut out); + self.events.publish_rx_ready(rx.published); + if rx.consumed == RX_IRQ_BUDGET { + out.budget_exhausted = true; + self.events.publish_service_needed(); + } + let sent = self.service_tx(core, TX_IRQ_BUDGET, &mut out); + self.events.publish_tx_space(sent); + if sent == TX_IRQ_BUDGET { + out.budget_exhausted = true; + self.events.publish_service_needed(); + } + if !out.budget_exhausted { + self.events.clear_service_needed(); + } } + self.events.finish_outcome(&mut out, &self.tx, &self.rx); out } @@ -596,14 +704,17 @@ impl SerialPort { match sample.flag { RxFlag::Normal => {} RxFlag::Break => { + self.events.publish_error(); self.counters.rx_breaks.fetch_add(1, Ordering::Relaxed); } RxFlag::Parity => { + self.events.publish_error(); self.counters .rx_parity_errors .fetch_add(1, Ordering::Relaxed); } RxFlag::Framing => { + self.events.publish_error(); self.counters .rx_framing_errors .fetch_add(1, Ordering::Relaxed); @@ -625,6 +736,7 @@ impl SerialPort { } if sample.overrun { + self.events.publish_error(); self.rx.overrun.fetch_add(1, Ordering::Relaxed); self.counters .rx_fifo_overruns @@ -1022,4 +1134,25 @@ mod tests { assert_eq!(outcome.rx_pushed, 2); assert!(!outcome.budget_exhausted); } + + #[test] + fn rx_budget_exhaustion_publishes_service_needed_snapshot() { + let mut uart = MockUart::new().irq(IrqSource::RX_DATA); + for index in 0..=RX_IRQ_BUDGET { + uart = uart.rx_byte(index as u8); + } + let parts = SerialPort::<8, 512>::split(uart, OwnerId(0)); + parts.port.startup(lease(), &Config::new()).unwrap(); + + let mut irq = parts.irq; + let outcome = irq.handle(lease()); + + assert!(outcome.claimed); + assert_eq!(outcome.rx_pushed, RX_IRQ_BUDGET); + assert!(outcome.budget_exhausted); + assert!(outcome.snapshot.rx_ready); + assert!(outcome.snapshot.service_needed); + assert!(outcome.snapshot.budget_exhausted); + assert!(outcome.snapshot.rx_seq > 0); + } } diff --git a/drivers/interface/rdif-serial/src/types.rs b/drivers/interface/rdif-serial/src/types.rs index ea125ef936..ff317a8c8b 100644 --- a/drivers/interface/rdif-serial/src/types.rs +++ b/drivers/interface/rdif-serial/src/types.rs @@ -88,6 +88,18 @@ pub struct SerialCounters { pub tx_bytes: u64, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SerialEventSnapshot { + pub rx_seq: u64, + pub tx_seq: u64, + pub rx_ready: bool, + pub tx_space: bool, + pub budget_exhausted: bool, + pub error: bool, + pub hangup: bool, + pub service_needed: bool, +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct SerialIrqOutcome { pub claimed: bool, @@ -95,4 +107,5 @@ pub struct SerialIrqOutcome { pub tx_sent: usize, pub tx_wakeup: bool, pub budget_exhausted: bool, + pub snapshot: SerialEventSnapshot, } diff --git a/net/ax-net/Cargo.toml b/net/ax-net/Cargo.toml index b19b758859..b88be61338 100644 --- a/net/ax-net/Cargo.toml +++ b/net/ax-net/Cargo.toml @@ -19,7 +19,7 @@ ax-io = { workspace = true } ax-kspin = { workspace = true } axpoll = { workspace = true } ax-sync = { workspace = true } -ax-task = { workspace = true, features = ["irq", "multitask"] } +ax-task = { workspace = true, features = ["multitask"] } bitflags = "2.9" cfg-if = { workspace = true } enum_dispatch = "0.3" diff --git a/net/ax-net/src/lib.rs b/net/ax-net/src/lib.rs index 34bd0bd213..f0f77ac508 100644 --- a/net/ax-net/src/lib.rs +++ b/net/ax-net/src/lib.rs @@ -78,7 +78,7 @@ use core::{ use ax_errno::{AxError, AxResult, ax_err_type}; use ax_sync::Mutex; -use ax_task::{IrqNotify, WaitQueue}; +use ax_task::{HardIrqSignal, WaitQueue}; use axpoll::{IoEvents, PollSet}; use smoltcp::{ socket::dns::{self, GetQueryResultError, StartQueryError}, @@ -156,7 +156,7 @@ impl Wake for DeferPollWake { static WIFI_CONTROLS: LazyLock>> = LazyLock::new(|| Mutex::new(Vec::new())); -static NET_IRQ_NOTIFY: IrqNotify = IrqNotify::new(); +static NET_IRQ_NOTIFY: HardIrqSignal = HardIrqSignal::new(); const DHCP_BOOTSTRAP_ATTEMPTS: usize = 200; const DHCP_BOOTSTRAP_POLL_INTERVAL: Duration = Duration::from_millis(10); @@ -706,7 +706,6 @@ pub fn reconfigure_wifi(name: &str, mode: WifiMode<'_>) -> AxResult<()> { /// task context. pub fn wake_net_task_irq() { NET_IRQ_NOTIFY.notify_irq(); - NET_POLL_WAKE.notify_one_from_irq(); } fn next_poll_delay() -> Duration { @@ -743,6 +742,7 @@ impl Wake for NetPollWake { fn net_poll_worker() { loop { let delay = next_poll_delay(); + NET_IRQ_NOTIFY.arm_current_task(); let timed_out = NET_POLL_WAKE.wait_timeout_until(delay, || { NET_POLL_REQUESTED.load(Ordering::Acquire) || NET_IRQ_NOTIFY.is_pending() diff --git a/os/StarryOS/kernel/Cargo.toml b/os/StarryOS/kernel/Cargo.toml index ca4ad8f5d7..219656d13e 100644 --- a/os/StarryOS/kernel/Cargo.toml +++ b/os/StarryOS/kernel/Cargo.toml @@ -56,7 +56,6 @@ axtest = [] [dependencies] ax-feat = { workspace = true, features = [ "fp-simd", - "irq", "uspace", "multitask", @@ -170,7 +169,7 @@ rbpf = { version = "0.4", default-features = false } [dev-dependencies] axtest.workspace = true ax-hal.workspace = true -ax-std = { workspace = true, features = ["ext-ld", "fs", "multitask", "irq"] } +ax-std = { workspace = true, features = ["ext-ld", "fs", "multitask"] } [[test]] name = "axtest_kernel" diff --git a/os/StarryOS/kernel/src/perf/bpf.rs b/os/StarryOS/kernel/src/perf/bpf.rs index c6a6013244..26b18b78a3 100644 --- a/os/StarryOS/kernel/src/perf/bpf.rs +++ b/os/StarryOS/kernel/src/perf/bpf.rs @@ -20,7 +20,7 @@ use ax_alloc::GlobalPage; use ax_errno::{AxError, AxResult}; use ax_hal::mem::virt_to_phys; use ax_memory_addr::{PAGE_SIZE_4K, PhysAddr}; -use ax_task::IrqNotify; +use ax_task::HardIrqSignal; use axpoll::{IoEvents, PollSet, Pollable}; use kbpf_basic::{ linux_bpf::{perf_event_mmap_page, perf_event_sample_format}, @@ -70,7 +70,7 @@ const BPF_JIT_MEM_PAGES: usize = 4; pub struct BpfPerfEventWrapper { inner: BpfPerfEvent, poll_ready: Arc, - poll_notify: Arc, + poll_notify: Arc, poll_alive: Arc, /// Weak handle to the contiguous pages backing the ringbuf. The strong /// ref(s) live in the user VMA(s); `strong_count() > 0` means a live @@ -83,7 +83,7 @@ impl BpfPerfEventWrapper { /// Construct the wrapper around a freshly-built `BpfPerfEvent`. pub fn new(inner: BpfPerfEvent) -> Self { let poll_ready = Arc::new(PollSet::new()); - let poll_notify = Arc::new(IrqNotify::new()); + let poll_notify = Arc::new(HardIrqSignal::new()); let poll_alive = Arc::new(AtomicBool::new(true)); start_bpf_perf_notify_worker(poll_ready.clone(), poll_notify.clone(), poll_alive.clone()); Self { @@ -129,7 +129,7 @@ impl Drop for BpfPerfEventWrapper { fn start_bpf_perf_notify_worker( poll_ready: Arc, - poll_notify: Arc, + poll_notify: Arc, poll_alive: Arc, ) { ax_task::spawn_with_name( diff --git a/os/StarryOS/kernel/src/perf/hw.rs b/os/StarryOS/kernel/src/perf/hw.rs index 011e2528bf..65b5818353 100644 --- a/os/StarryOS/kernel/src/perf/hw.rs +++ b/os/StarryOS/kernel/src/perf/hw.rs @@ -37,7 +37,7 @@ use ax_hal::mem::virt_to_phys; #[cfg(target_arch = "aarch64")] use ax_memory_addr::PhysAddr; #[cfg(target_arch = "aarch64")] -use ax_task::IrqNotify; +use ax_task::HardIrqSignal; #[cfg(target_arch = "aarch64")] use axpoll::PollSet; use axpoll::{IoEvents, Pollable}; @@ -204,10 +204,10 @@ impl RingState { /// Sampling state attached to a `HwPerfEvent` when `attr.sample_period > 0`. /// /// Holds the period and `sample_type`, the deferred poll machinery (mirroring -/// [`super::bpf::BpfPerfEventWrapper`]: a `PollSet` woken by an `IrqNotify` via a +/// [`super::bpf::BpfPerfEventWrapper`]: a `PollSet` woken by an `HardIrqSignal` via a /// background worker), and — once `mmap(perf_fd)` runs — the ring buffer. /// -/// The `notify` `Arc` is the strong reference that keeps the `IrqNotify` alive +/// The `notify` `Arc` is the strong reference that keeps the `HardIrqSignal` alive /// for the registered [`SampleSlot`]'s raw pointer (see [`super::sampling`]): /// teardown unregisters the slot before this `SamplingState` (and thus the /// `Arc`) drops. @@ -226,7 +226,7 @@ struct SamplingState { /// Readiness set readers wait on; woken (with `IoEvents::IN`) by the worker. poll_ready: Arc, /// IRQ-safe notification the overflow handler pokes; drained by the worker. - notify: Arc, + notify: Arc, /// Liveness flag for the worker; cleared on drop to stop it. poll_alive: Arc, /// The ring buffer pages, `Some` after the first `mmap(perf_fd)`. @@ -255,7 +255,7 @@ impl core::fmt::Debug for SamplingState { #[cfg(target_arch = "aarch64")] fn start_sampling_notify_worker( poll_ready: Arc, - notify: Arc, + notify: Arc, poll_alive: Arc, ) { ax_task::spawn_with_name( @@ -384,7 +384,7 @@ impl HwPerfEvent { /// 3. clear the per-CPU `SampleSlot` (`unregister`) — the handler can no /// longer reach the `notify` pointer, /// - /// after which it is safe for the owning `Arc` / `Arc` + /// after which it is safe for the owning `Arc` / `Arc` /// to drop. Idempotent: safe to call from both `disable` and `Drop`. fn teardown_sampling_irq(&self) { if self.sampling.is_none() { @@ -456,7 +456,7 @@ impl Drop for HwPerfEvent { return; } // For sampling events, mask the IRQ, stop the counter, and clear the - // registry slot BEFORE the `Arc`/`Arc` held in + // registry slot BEFORE the `Arc`/`Arc` held in // `sampling` drop, so the overflow handler can never dereference a // freed `notify` pointer or write into freed ring pages. self.teardown_sampling_irq(); @@ -807,7 +807,7 @@ fn device_mmap_per_task( // Spawn the deferred worker (mirrors the M2 path): it turns IRQ-context // `notify_irq` pokes into `axpoll` `IoEvents::IN` wakeups. let poll_ready = Arc::new(PollSet::new()); - let notify = Arc::new(IrqNotify::new()); + let notify = Arc::new(HardIrqSignal::new()); let poll_alive = Arc::new(AtomicBool::new(true)); start_sampling_notify_worker(poll_ready.clone(), notify.clone(), poll_alive.clone()); @@ -961,7 +961,7 @@ pub fn perf_event_open_hw(attr: &perf_event_attr, pid: i32) -> AxResult` for its entire life, and teardown +//! `Arc` for its entire life, and teardown //! ([`super::hw::HwPerfEvent`]'s disable/Drop) calls [`unregister`] — clearing //! the slot — *before* dropping that `Arc`. The handler therefore only ever //! dereferences a pointer whose target is still alive. @@ -45,7 +45,7 @@ use core::sync::atomic::Ordering; use ax_hal::irq::{IrqContext, IrqId, IrqReturn}; use ax_kernel_guard::NoPreemptIrqSave; -use ax_task::IrqNotify; +use ax_task::HardIrqSignal; use kbpf_basic::linux_bpf::perf_event_mmap_page; fn pmu_irq() -> Result { @@ -172,8 +172,8 @@ pub struct SampleSlot { /// fields. `0` when the event was opened without per-event ids (the common /// case in this single-group implementation). pub id: u64, - /// Raw pointer to the owning event's [`IrqNotify`], woken after each sample. - /// Kept alive by the event's strong `Arc` for as long as the slot + /// Raw pointer to the owning event's [`HardIrqSignal`], woken after each sample. + /// Kept alive by the event's strong `Arc` for as long as the slot /// is registered (see module docs). pub notify: *const (), /// Frequency mode (`attr.freq`): after each sample re-derive [`period`](Self::period) @@ -230,7 +230,7 @@ pub fn register(n: usize, slot: SampleSlot) { /// Clears the sampling slot for programmable counter `n` on the current CPU. /// /// Mirror of [`register`]. Teardown calls this *before* the owning event drops -/// its `Arc`, so once this returns the handler can no longer reach a +/// its `Arc`, so once this returns the handler can no longer reach a /// stale `notify` pointer for counter `n`. pub fn unregister(n: usize) { if n > MAX_COUNTER { @@ -400,9 +400,9 @@ pub fn pmu_overflow_handler(_ctx: IrqContext) -> IrqReturn { // leader's ring but has no notify of its own — its `notify` is null, and // the leader's own poller re-checks `data_head` on its next poll. The // pointer, when non-null, is valid: the owning event holds the backing - // `Arc` while registered (see the module-level soundness note). + // `Arc` while registered (see the module-level soundness note). if !notify_ptr.is_null() { - let notify = unsafe { &*(notify_ptr as *const IrqNotify) }; + let notify = unsafe { &*(notify_ptr as *const HardIrqSignal) }; notify.notify_irq(); } } diff --git a/os/StarryOS/kernel/src/perf/task.rs b/os/StarryOS/kernel/src/perf/task.rs index ef3efdeafd..0b924d33d7 100644 --- a/os/StarryOS/kernel/src/perf/task.rs +++ b/os/StarryOS/kernel/src/perf/task.rs @@ -70,7 +70,7 @@ use core::{ use ax_alloc::GlobalPage; use ax_kspin::SpinNoIrq; use ax_runtime::hal::paging::MappingFlags; -use ax_task::IrqNotify; +use ax_task::HardIrqSignal; use super::{ hw, @@ -188,9 +188,9 @@ pub struct PerTaskCounter { ring_vaddr: AtomicUsize, /// Total ring length in bytes (header page + data region); `0` until mapped. ring_len: AtomicUsize, - /// Raw pointer to the live [`IrqNotify`], or null until mapped. Copied into + /// Raw pointer to the live [`HardIrqSignal`], or null until mapped. Copied into /// the [`SampleSlot`] in [`perf_sched_in`] so the overflow handler can wake - /// the poll worker. Kept alive by the `Arc` in [`SamplingAnchors`] + /// the poll worker. Kept alive by the `Arc` in [`SamplingAnchors`] /// for as long as a slot may reference it (the slot is unregistered before /// the `Arc` drops in [`free_hw`]). notify_ptr: AtomicUsize, @@ -227,7 +227,7 @@ struct SamplingAnchors { ring_pages: Arc, /// IRQ-safe notification the overflow handler pokes; drained by the worker. /// Holding this `Arc` keeps `notify_ptr` valid for the registered slot. - notify: Arc, + notify: Arc, /// Readiness set the perf fd's poller waits on; woken (`IoEvents::IN`) by the /// worker after each sample lands in the ring. poll_ready: Arc, @@ -382,7 +382,7 @@ impl PerTaskCounter { ring_pages: Arc, ring_vaddr: usize, ring_len: usize, - notify: Arc, + notify: Arc, poll_ready: Arc, poll_alive: Arc, ) { @@ -1008,7 +1008,7 @@ pub fn on_task_exit(thr: &Thread) { /// torn down in the UAF-safe order before the slot/ring `Arc`s drop: stop the /// counter, mask the IRQ, then `unregister` the [`SampleSlot`] — so the overflow /// handler can no longer reach the ring or `notify` pointer. Only after that are -/// the [`SamplingAnchors`] (the `Arc` ring + `Arc`) +/// the [`SamplingAnchors`] (the `Arc` ring + `Arc`) /// dropped and the worker stopped. pub fn free_hw(ptc: &PerTaskCounter) { if ptc diff --git a/os/StarryOS/kernel/src/pseudofs/dev/kpu.rs b/os/StarryOS/kernel/src/pseudofs/dev/kpu.rs index 60134969a2..a036c8e0db 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/kpu.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/kpu.rs @@ -1,13 +1,13 @@ use core::{ any::Any, mem::{MaybeUninit, size_of}, - sync::atomic::{AtomicU64, Ordering}, + sync::atomic::{AtomicBool, AtomicU64, Ordering}, time::Duration, }; use ax_memory_addr::{PhysAddr, PhysAddrRange}; use ax_runtime::hal::cpu::asm::user_copy; -use ax_task::WaitQueue; +use ax_task::{HardIrqSignal, WaitQueue}; use axfs_ng_vfs::{DeviceId, NodeFlags, VfsError, VfsResult}; use k230_kpu::{ CommandRange, KPU_CFG_PADDR, KPU_CFG_SIZE, KPU_INFO_F_FAKE_OUTPUT, KPU_INFO_F_FDT, @@ -29,6 +29,8 @@ const KPU_IRQ_WAIT_TIMEOUT: Duration = Duration::from_millis(100); // K230 exposes one KPU instance. If a future platform exposes more instances, // move this IRQ state into per-device storage. static KPU_IRQ_COUNT: AtomicU64 = AtomicU64::new(0); +static KPU_IRQ_NOTIFY: HardIrqSignal = HardIrqSignal::new(); +static KPU_IRQ_WORKER_STARTED: AtomicBool = AtomicBool::new(false); static KPU_DONE_WQ: WaitQueue = WaitQueue::new(); pub struct KpuDevice { @@ -85,6 +87,7 @@ impl KpuDevice { return Ok(()); } if self.irq_registration.is_some() { + ensure_kpu_irq_worker(); let timed_out = KPU_DONE_WQ.wait_timeout_until(KPU_IRQ_WAIT_TIMEOUT, || self.hw.is_done()); if !timed_out { @@ -448,10 +451,24 @@ fn register_kpu_irq(irq: ax_runtime::hal::irq::IrqId) -> Option fn kpu_irq_handler(_ctx: ax_runtime::hal::irq::IrqContext) -> ax_runtime::hal::irq::IrqReturn { KPU_IRQ_COUNT.fetch_add(1, Ordering::AcqRel); - KPU_DONE_WQ.notify_all_from_irq(); + KPU_IRQ_NOTIFY.notify_irq(); ax_runtime::hal::irq::IrqReturn::Handled } +fn ensure_kpu_irq_worker() { + if KPU_IRQ_WORKER_STARTED.swap(true, Ordering::AcqRel) { + return; + } + + ax_task::spawn_with_name( + || loop { + KPU_IRQ_NOTIFY.wait(); + KPU_DONE_WQ.notify_all_deferred(); + }, + "kpu-irq".into(), + ); +} + fn fallback_irq() -> Option { None } diff --git a/os/StarryOS/kernel/src/pseudofs/dev/tty/serial.rs b/os/StarryOS/kernel/src/pseudofs/dev/tty/serial.rs index 3e95b64da5..3edc43f176 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/tty/serial.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/tty/serial.rs @@ -1,6 +1,6 @@ use alloc::{format, string::String, sync::Arc, vec, vec::Vec}; use core::{ - sync::atomic::{AtomicBool, AtomicU32, Ordering}, + sync::atomic::{AtomicBool, AtomicU64, Ordering}, time::Duration, }; @@ -16,11 +16,11 @@ use ax_runtime::hal::{ irq::{AutoEnable, CpuId, IrqAffinity, IrqHandle, IrqId, IrqRequest, ShareMode}, }; use ax_sync::Mutex; -use ax_task::IrqNotify; +use ax_task::{HardIrqSignal, HardIrqWaker, local::RuntimeEvent}; use axpoll::{IoEvents, PollSet}; use bitflags::bitflags; use rdrive::DeviceId as RDriveDeviceId; -use spin::LazyLock; +use spin::{LazyLock, Once}; use starry_process::Process; use super::{ @@ -45,6 +45,7 @@ bitflags! { const RX_READY = 1 << 0; const TX_SPACE = 1 << 1; const HANGUP = 1 << 2; + const RESERVICE = 1 << 3; } } @@ -85,45 +86,99 @@ struct SerialBackend { events: SerialEvents, input_source: Arc, output_source: Arc, - tx_notify: IrqNotify, + tx_notify: HardIrqSignal, output_lock: Mutex<()>, } struct SerialEvents { - pending: AtomicU32, - notify: IrqNotify, + event: RuntimeEvent, + irq_waker: Once, + pending_hint: AtomicBool, + rx_seq: AtomicU64, + tx_seq: AtomicU64, } impl SerialEvents { const fn new() -> Self { Self { - pending: AtomicU32::new(0), - notify: IrqNotify::new(), + event: RuntimeEvent::new(), + irq_waker: Once::new(), + pending_hint: AtomicBool::new(false), + rx_seq: AtomicU64::new(0), + tx_seq: AtomicU64::new(0), } } + fn init_irq_waker(&self, waker: HardIrqWaker) { + self.irq_waker.call_once(|| waker); + } + fn publish_irq(&self, events: SerialEventBits) { if events.is_empty() { return; } - self.pending.fetch_or(events.bits(), Ordering::Release); - self.notify.notify_irq(); + self.pending_hint.store(true, Ordering::Release); + let bits = u64::from(events.bits()); + if let Some(waker) = self.irq_waker.get() { + let _ = self.event.publish_from_irq_with(bits, waker); + } else { + let _ = self.event.publish_from_irq(bits); + } } fn publish(&self, events: SerialEventBits) { if events.is_empty() { return; } - self.pending.fetch_or(events.bits(), Ordering::Release); - self.notify.notify(); + self.pending_hint.store(true, Ordering::Release); + self.event.publish(u64::from(events.bits())); } fn wait(&self) { - self.notify.wait(); + self.event + .wait_until(|| self.pending_hint.load(Ordering::Acquire)); } fn take(&self) -> SerialEventBits { - SerialEventBits::from_bits_retain(self.pending.swap(0, Ordering::AcqRel)) + self.pending_hint.store(false, Ordering::Release); + let bits = self.event.take_bits(); + SerialEventBits::from_bits_retain(bits as u32) + } + + fn wake_waiters_deferred(&self) { + self.event.wake_waiters_deferred(); + } + + fn observe_rx_seq(&self, seq: u64) -> bool { + let mut current = self.rx_seq.load(Ordering::Acquire); + while seq > current { + match self.rx_seq.compare_exchange_weak( + current, + seq, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return true, + Err(next) => current = next, + } + } + false + } + + fn observe_tx_seq(&self, seq: u64) -> bool { + let mut current = self.tx_seq.load(Ordering::Acquire); + while seq > current { + match self.tx_seq.compare_exchange_weak( + current, + seq, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return true, + Err(next) => current = next, + } + } + false } } @@ -318,7 +373,7 @@ fn new_serial_tty(number: usize, serial: SerialDevice) -> AxResult Config { fn spawn_serial_event_worker(backend: Arc) { let task_name = format!("{}-event", backend.tty_name); ax_task::spawn_with_name( - move || loop { - backend.events.wait(); + move || { + backend + .events + .init_irq_waker(ax_task::current_hard_irq_waker()); loop { - let pending = backend.events.take(); - if pending.is_empty() { - break; - } - if pending.contains(SerialEventBits::RX_READY) { - unsafe { backend.input_source.wake(IoEvents::IN) }; - } - if pending.contains(SerialEventBits::TX_SPACE) { - backend.tx_notify.notify(); - unsafe { backend.output_source.wake(IoEvents::OUT) }; - let outcome = backend.service_on_owner(SerialSoftWork::TX_KICK); - publish_serial_outcome(&backend, outcome, false); + backend.events.wait(); + loop { + let pending = backend.events.take(); + if pending.is_empty() { + break; + } + backend.events.wake_waiters_deferred(); + if pending.contains(SerialEventBits::RX_READY) { + unsafe { backend.input_source.wake(IoEvents::IN) }; + } + if pending.contains(SerialEventBits::TX_SPACE) { + backend.tx_notify.notify(); + unsafe { backend.output_source.wake(IoEvents::OUT) }; + let outcome = backend.service_on_owner(SerialSoftWork::TX_KICK); + publish_serial_outcome(&backend, outcome, false); + } + if pending.contains(SerialEventBits::RESERVICE) { + let outcome = backend.service_on_owner(SerialSoftWork::RESERVICE); + publish_serial_outcome(&backend, outcome, false); + } } } }, @@ -561,12 +626,18 @@ fn publish_serial_outcome( from_irq: bool, ) -> SerialEventBits { let mut events = SerialEventBits::empty(); - if outcome.rx_pushed > 0 { + if outcome.rx_pushed > 0 || backend.events.observe_rx_seq(outcome.snapshot.rx_seq) { events |= SerialEventBits::RX_READY; } - if outcome.tx_wakeup { + if outcome.tx_wakeup || backend.events.observe_tx_seq(outcome.snapshot.tx_seq) { events |= SerialEventBits::TX_SPACE; } + if outcome.snapshot.hangup { + events |= SerialEventBits::HANGUP; + } + if outcome.budget_exhausted || outcome.snapshot.service_needed { + events |= SerialEventBits::RESERVICE; + } if from_irq { backend.events.publish_irq(events); diff --git a/os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs b/os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs index 3edd5d0048..e80029cbf5 100644 --- a/os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs +++ b/os/StarryOS/kernel/src/pseudofs/usbfs/manager.rs @@ -9,7 +9,7 @@ use ax_errno::{AxError, AxResult, LinuxError}; use ax_kspin::{SpinNoIrq as Mutex, SpinRwLock as RwLock}; use ax_runtime::hal::irq::IrqId; use ax_sync::Mutex as BlockingMutex; -use ax_task::IrqNotify; +use ax_task::HardIrqSignal; use crab_usb::{ Device, DeviceInfo, Endpoint, ProbedDevice, usb_if::{ @@ -233,7 +233,7 @@ pub(super) struct UsbFsManager { state: Mutex, open_lock: BlockingMutex<()>, usb_activity: UsbActivity, - irq_notify: IrqNotify, + irq_notify: HardIrqSignal, } struct UsbActivity { @@ -390,7 +390,7 @@ impl UsbFsManager { state: Mutex::new(UsbFsState { hosts, devices }), open_lock: BlockingMutex::new(()), usb_activity: UsbActivity::new(), - irq_notify: IrqNotify::new(), + irq_notify: HardIrqSignal::new(), } } diff --git a/os/StarryOS/kernel/src/syscall/task/exit.rs b/os/StarryOS/kernel/src/syscall/task/exit.rs index a6c07f17ae..acbb768704 100644 --- a/os/StarryOS/kernel/src/syscall/task/exit.rs +++ b/os/StarryOS/kernel/src/syscall/task/exit.rs @@ -4,10 +4,10 @@ use crate::task::do_exit; pub fn sys_exit(exit_code: i32) -> AxResult { do_exit(exit_code << 8, false); - Ok(0) + ax_task::exit(0) } pub fn sys_exit_group(exit_code: i32) -> AxResult { do_exit(exit_code << 8, true); - Ok(0) + ax_task::exit(0) } diff --git a/os/StarryOS/kernel/src/task/mod.rs b/os/StarryOS/kernel/src/task/mod.rs index 98cda01c97..2217e9615b 100644 --- a/os/StarryOS/kernel/src/task/mod.rs +++ b/os/StarryOS/kernel/src/task/mod.rs @@ -637,6 +637,8 @@ pub struct ProcessData { /// The child exit wait event pub child_exit_event: Arc, + /// Scheduler wait queue used by waitpid/waitid. + pub child_wait_queue: ax_task::WaitQueue, /// Self exit event pub exit_event: Arc, /// Woken every time a thread in this process exits. Used by a thread @@ -834,6 +836,7 @@ impl ProcessData { rlim: RwLock::default(), child_exit_event: Arc::default(), + child_wait_queue: ax_task::WaitQueue::new(), exit_event: Arc::default(), thread_exit_event: Arc::default(), exec_lock: Mutex::new(()), diff --git a/os/StarryOS/kernel/src/task/ops.rs b/os/StarryOS/kernel/src/task/ops.rs index d7c4bec751..57e445294e 100644 --- a/os/StarryOS/kernel/src/task/ops.rs +++ b/os/StarryOS/kernel/src/task/ops.rs @@ -569,9 +569,12 @@ pub fn do_exit(exit_code: i32, group_exit: bool) { let table = futex_table_for_process(&thr.proc_data, &key); let guard = table.get(&key); if let Some(futex) = guard { + // Wake pthread joiners, but do not yield here. The process zombie + // state is not published until `process.exit()` below, so yielding + // in the middle of exit can leave the parent unable to observe this + // task while the exiting thread waits to be scheduled again. futex.wq.wake(1, u32::MAX); } - ax_task::yield_now(); } let process = &thr.proc_data.proc; @@ -630,6 +633,7 @@ pub fn do_exit(exit_code: i32, group_exit: bool) { wait_parent_tid, ); process.exit(); + thr.set_exit(); if let Some(parent) = process.parent() { if let Some(signo) = thr.proc_data.exit_signal { use starry_signal::Signo; @@ -646,6 +650,7 @@ pub fn do_exit(exit_code: i32, group_exit: bool) { } if let Ok(data) = get_process_data(parent.pid()) { // Child exit state is published before waking waiters. + data.child_wait_queue.notify_all(true); unsafe { data.child_exit_event.wake(axpoll::IoEvents::IN) }; } } @@ -656,6 +661,7 @@ pub fn do_exit(exit_code: i32, group_exit: bool) { && let Ok(data) = get_process_data(tracer_pid) { // Child exit state is published before waking waiters. + data.child_wait_queue.notify_all(true); unsafe { data.child_exit_event.wake(axpoll::IoEvents::IN) }; } // Send pdeathsig to child processes @@ -686,10 +692,9 @@ pub fn do_exit(exit_code: i32, group_exit: bool) { thr.proc_data.release_aspace_slot_if_needed(); } // Thread exit state is published before waking waiters. + thr.set_exit(); unsafe { thr.exit_event.wake(axpoll::IoEvents::IN) }; unsafe { thr.proc_data.thread_exit_event.wake(axpoll::IoEvents::IN) }; - - thr.set_exit(); } /// Rebinds a task's user-visible TID in [`TASK_TABLE`] from `old_tid` to diff --git a/os/StarryOS/kernel/src/task/signal.rs b/os/StarryOS/kernel/src/task/signal.rs index a612c92512..b29995f186 100644 --- a/os/StarryOS/kernel/src/task/signal.rs +++ b/os/StarryOS/kernel/src/task/signal.rs @@ -289,6 +289,7 @@ fn notify_ptrace_waiter(thr: &Thread, signo: Signo) { ); let _ = send_signal_to_process(waiter_pid, Some(sigchld)); // Ptrace stop report is published before waking waiters. + parent_data.child_wait_queue.notify_all(true); unsafe { parent_data.child_exit_event.wake(axpoll::IoEvents::IN) }; } } @@ -419,6 +420,7 @@ fn notify_parent_job_change(proc_data: &ProcessData, code: i32, status: i32) { let _ = send_signal_to_process(parent.pid(), Some(sig)); if let Ok(data) = get_process_data(parent.pid()) { // Job-control report is published before waking waiters. + data.child_wait_queue.notify_all(true); unsafe { data.child_exit_event.wake(axpoll::IoEvents::IN) }; } } diff --git a/os/StarryOS/kernel/src/tracepoint/mod.rs b/os/StarryOS/kernel/src/tracepoint/mod.rs index 8ede91f25e..225a931ddf 100644 --- a/os/StarryOS/kernel/src/tracepoint/mod.rs +++ b/os/StarryOS/kernel/src/tracepoint/mod.rs @@ -17,7 +17,7 @@ use ax_lazyinit::LazyInit; use ax_memory_addr::VirtAddr; use ax_runtime::hal::{percpu::this_cpu_id, time::monotonic_time_nanos}; use ax_sync::Mutex; -use ax_task::{IrqNotify, current}; +use ax_task::{HardIrqSignal, current}; use axfs_ng_vfs::NodePermission; use axpoll::{IoEvents, PollSet}; use ktracepoint::*; @@ -67,7 +67,7 @@ struct TraceState { point_map: LazyInit>, raw_pipe: Mutex, pipe_event: PollSet, - pipe_notify: IrqNotify, + pipe_notify: HardIrqSignal, cmdline_cache: LazyInit>, ext_tracepoints: LazyInit>, } @@ -78,7 +78,7 @@ impl TraceState { point_map: LazyInit::new(), raw_pipe: Mutex::new(TracePipeRaw::new(TRACE_RAW_PIPE_CAPACITY)), pipe_event: PollSet::new(), - pipe_notify: IrqNotify::new(), + pipe_notify: HardIrqSignal::new(), cmdline_cache: LazyInit::new(), ext_tracepoints: LazyInit::new(), } diff --git a/os/StarryOS/lkm/hello/Cargo.toml b/os/StarryOS/lkm/hello/Cargo.toml index d1c5469ffa..06f55caa01 100644 --- a/os/StarryOS/lkm/hello/Cargo.toml +++ b/os/StarryOS/lkm/hello/Cargo.toml @@ -10,19 +10,18 @@ categories.workspace = true publish = false [dependencies] -ax-feat = { workspace = true, features = ["ext-ld", "fs-times", "irq"] } +ax-feat = { workspace = true, features = ["ext-ld", "fs-times"] } ax-driver.workspace = true ax-hal.workspace = true axplat-dyn = { workspace = true, optional = true } starry-kernel = { workspace = true, features = ["dev-log", "ext4"] } -ax-std = { workspace = true, features = ["ext-ld", "fs", "multitask", "irq"] } +ax-std = { workspace = true, features = ["ext-ld", "fs", "multitask"] } kmod-tools.workspace = true [features] default = [] qemu = [ - "ax-feat/irq", "ax-feat/rtc", "ax-feat/display", "starry-kernel/input", diff --git a/os/StarryOS/lkm/kprobe_test/Cargo.toml b/os/StarryOS/lkm/kprobe_test/Cargo.toml index c739d7c481..69cf997464 100644 --- a/os/StarryOS/lkm/kprobe_test/Cargo.toml +++ b/os/StarryOS/lkm/kprobe_test/Cargo.toml @@ -10,12 +10,12 @@ categories.workspace = true publish = false [dependencies] -ax-feat = { workspace = true, features = ["ext-ld", "fs-times", "irq"] } +ax-feat = { workspace = true, features = ["ext-ld", "fs-times"] } ax-driver.workspace = true ax-hal.workspace = true axplat-dyn = { workspace = true, optional = true } starry-kernel = { workspace = true, features = ["dev-log", "ext4"] } -ax-std = { workspace = true, features = ["ext-ld", "fs", "multitask", "irq"] } +ax-std = { workspace = true, features = ["ext-ld", "fs", "multitask"] } ax-log.workspace = true kmod-tools.workspace = true @@ -24,7 +24,6 @@ kprobe.workspace = true [features] default = [] qemu = [ - "ax-feat/irq", "ax-feat/rtc", "ax-feat/display", "starry-kernel/input", diff --git a/os/StarryOS/starryos/Cargo.toml b/os/StarryOS/starryos/Cargo.toml index 6898d6f77e..3646ed5e78 100644 --- a/os/StarryOS/starryos/Cargo.toml +++ b/os/StarryOS/starryos/Cargo.toml @@ -16,7 +16,6 @@ version = "0.5.20" [features] default = [] qemu = [ - "ax-feat/irq", "ax-feat/rtc", "ax-feat/display", "starry-kernel/input", @@ -45,8 +44,8 @@ name = "xtask" path = "xtask/main.rs" [dependencies] -ax-feat = { workspace = true, features = ["ext-ld", "fs-times", "irq"] } -ax-std = { workspace = true, features = ["ext-ld", "fs", "multitask", "irq"] } +ax-feat = { workspace = true, features = ["ext-ld", "fs-times"] } +ax-std = { workspace = true, features = ["ext-ld", "fs", "multitask"] } ax-driver.workspace = true ax-hal.workspace = true axplat-dyn.workspace = true diff --git a/os/arceos/api/arceos_api/Cargo.toml b/os/arceos/api/arceos_api/Cargo.toml index d8850d25a2..60193ef49a 100644 --- a/os/arceos/api/arceos_api/Cargo.toml +++ b/os/arceos/api/arceos_api/Cargo.toml @@ -10,7 +10,6 @@ license.workspace = true [features] default = [] -irq = ["ax-feat/irq"] ipi = ["dep:ax-ipi", "ax-feat/ipi"] alloc = ["dep:ax-alloc", "ax-feat/alloc"] paging = ["dep:ax-mm", "ax-feat/paging"] diff --git a/os/arceos/api/arceos_api/src/imp/task.rs b/os/arceos/api/arceos_api/src/imp/task.rs index 8cac3497fc..884719d3b4 100644 --- a/os/arceos/api/arceos_api/src/imp/task.rs +++ b/os/arceos/api/arceos_api/src/imp/task.rs @@ -11,11 +11,7 @@ pub fn ax_yield_now() { #[cfg(feature = "multitask")] ax_task::yield_now(); #[cfg(not(feature = "multitask"))] - if cfg!(feature = "irq") { - ax_hal::asm::wait_for_irqs(); - } else { - core::hint::spin_loop(); - } + ax_hal::asm::wait_for_irqs(); } #[track_caller] @@ -111,14 +107,10 @@ cfg_task! { #[track_caller] pub fn ax_wait_queue_wait(wq: &AxWaitQueueHandle, timeout: Option) -> bool { - #[cfg(feature = "irq")] if let Some(dur) = timeout { return wq.0.wait_timeout(dur); } - if timeout.is_some() { - ax_log::warn!("ax_wait_queue_wait: the `timeout` argument is ignored without the `irq` feature"); - } wq.0.wait(); false } @@ -129,14 +121,10 @@ cfg_task! { until_condition: impl Fn() -> bool, timeout: Option, ) -> bool { - #[cfg(feature = "irq")] if let Some(dur) = timeout { return wq.0.wait_timeout_until(dur, until_condition); } - if timeout.is_some() { - ax_log::warn!("ax_wait_queue_wait_until: the `timeout` argument is ignored without the `irq` feature"); - } wq.0.wait_until(until_condition); false } diff --git a/os/arceos/api/arceos_posix_api/Cargo.toml b/os/arceos/api/arceos_posix_api/Cargo.toml index 2bf6b52961..815123b29d 100644 --- a/os/arceos/api/arceos_posix_api/Cargo.toml +++ b/os/arceos/api/arceos_posix_api/Cargo.toml @@ -19,7 +19,6 @@ license.workspace = true default = [] smp = ["ax-feat/smp"] -irq = ["ax-feat/irq"] alloc = ["dep:ax-alloc", "ax-feat/alloc"] multitask = ["alloc", "ax-task/multitask", "ax-feat/multitask", "ax-sync/multitask"] lockdep = ["multitask", "ax-feat/lockdep", "ax-sync/lockdep", "ax-kspin/lockdep"] diff --git a/os/arceos/api/arceos_posix_api/src/imp/task.rs b/os/arceos/api/arceos_posix_api/src/imp/task.rs index e8e8720b19..316ad1a621 100644 --- a/os/arceos/api/arceos_posix_api/src/imp/task.rs +++ b/os/arceos/api/arceos_posix_api/src/imp/task.rs @@ -9,11 +9,7 @@ pub fn sys_sched_yield() -> c_int { #[cfg(feature = "multitask")] ax_task::yield_now(); #[cfg(not(feature = "multitask"))] - if cfg!(feature = "irq") { - ax_hal::asm::wait_for_irqs(); - } else { - core::hint::spin_loop(); - } + ax_hal::asm::wait_for_irqs(); 0 } diff --git a/os/arceos/api/axfeat/Cargo.toml b/os/arceos/api/axfeat/Cargo.toml index ece28c361e..b2da058d48 100644 --- a/os/arceos/api/axfeat/Cargo.toml +++ b/os/arceos/api/axfeat/Cargo.toml @@ -22,9 +22,10 @@ uspace = ["ax-hal/uspace", "ax-task?/uspace"] hv = ["ax-hal/hv"] # Interrupts -irq = ["ax-hal/irq", "ax-runtime/irq", "ax-task?/irq", "ax-driver?/irq"] -ipi = ["irq", "dep:ax-ipi", "ax-hal/ipi", "ax-runtime/ipi", "ax-task?/ipi"] -wake-ipi = ["irq", "ax-hal/ipi", "ax-runtime/wake-ipi"] +ipi = ["dep:ax-ipi", "ax-hal/ipi", "ax-runtime/ipi", "ax-task?/ipi"] +# Lightweight hardware IPI used to wake CPUs for IRQ epilogue wake-queue drain. +# It intentionally does not enable the full ax-ipi remote callback queue. +wake-ipi = ["ax-hal/ipi", "ax-runtime/wake-ipi", "ax-task?/irq-wake-ipi"] ext-ld = ["ax-runtime/ext-ld"] @@ -49,8 +50,8 @@ lockdep = [ task-ext = ["ax-task/task-ext"] tracepoint-hooks = ["ax-task/tracepoint-hooks"] sched-fifo = ["ax-task/sched-fifo"] -sched-rr = ["ax-task/sched-rr", "irq"] -sched-cfs = ["ax-task/sched-cfs", "irq"] +sched-rr = ["ax-task/sched-rr"] +sched-cfs = ["ax-task/sched-cfs"] stack-guard-page = [ "ipi", "multitask", @@ -73,7 +74,6 @@ fs-times = ["fs", "ax-fs-ng/times"] # Networking net = [ "paging", - "irq", "multitask", "dep:ax-net", "ax-runtime/net", @@ -96,7 +96,7 @@ input = [ ] # USB -usb = ["irq", "ax-driver?/usb"] +usb = ["ax-driver?/usb"] # Real Time Clock (RTC) Driver. rtc = ["ax-hal/rtc", "ax-runtime/rtc"] diff --git a/os/arceos/api/axfeat/src/lib.rs b/os/arceos/api/axfeat/src/lib.rs index 603b3e4fff..2f91ad853d 100644 --- a/os/arceos/api/axfeat/src/lib.rs +++ b/os/arceos/api/axfeat/src/lib.rs @@ -6,7 +6,6 @@ //! - `smp`: Enable SMP (symmetric multiprocessing) support. //! - `fp-simd`: Enable floating point and SIMD support. //! - Interrupts: -//! - `irq`: Enable interrupt handling support. //! - `ipi`: Enable Inter-Processor Interrupts (IPIs). //! - Memory //! - `alloc`: Enable dynamic memory allocation. diff --git a/os/arceos/doc/std_support_readme.md b/os/arceos/doc/std_support_readme.md index 656ff9ba8f..2af7e0a30e 100644 --- a/os/arceos/doc/std_support_readme.md +++ b/os/arceos/doc/std_support_readme.md @@ -45,7 +45,7 @@ An app declares the ArceOS-side features it needs behind its app-local ```toml [features] default = [] -arceos = ["dep:ax-std", "ax-std/fs", "ax-std/net", "ax-std/multitask", "ax-std/irq"] +arceos = ["dep:ax-std", "ax-std/fs", "ax-std/net", "ax-std/multitask"] [dependencies] ax-std = { workspace = true, optional = true } diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/device.rs b/os/arceos/modules/axfs-ng/src/block_runtime/device.rs index 21cb364e10..437cbfff04 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/device.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/device.rs @@ -10,7 +10,7 @@ use irq_framework::IrqId; use rdif_block::{ BlkError, CompletionHint, CompletionSink as RdifCompletionSink, DeviceInfo, IQueue, OwnedRequest, QueueHandle, QueueInfo, Request, RequestFlags, RequestId, RequestOp, - RequestPoll as OwnedRequestPoll, RequestStatus, TransferChunk, TransferPlanner, + RequestPoll as OwnedRequestPoll, RequestStatus, RequestToken, TransferChunk, TransferPlanner, TransferRuntimeCaps, validate_request, }; @@ -202,6 +202,13 @@ impl RuntimeQueue { Self::Owned(queue) => queue.info(), } } + + fn request_token(&self, request: RequestId) -> Option { + match self { + Self::Legacy(queue) => queue.request_token(request), + Self::Owned(queue) => queue.request_token(request), + } + } } impl QueueRuntime { @@ -232,14 +239,14 @@ impl QueueRuntime { pub struct BlockRuntime { devices: Vec>, - irq_registrations: Vec>, + irq_registrations: SpinNoIrq>>, } impl BlockRuntime { pub fn new() -> Self { Self { devices: Vec::new(), - irq_registrations: Vec::new(), + irq_registrations: SpinNoIrq::new(Vec::new()), } } @@ -251,8 +258,8 @@ impl BlockRuntime { &self.devices } - pub fn push_irq_registration(&mut self, registration: Box) { - self.irq_registrations.push(registration); + pub fn push_irq_registration(&self, registration: Box) { + self.irq_registrations.lock().push(registration); } } @@ -356,32 +363,76 @@ impl BlockIrqAction { impl BlockRuntime { pub fn from_rdif_devices(devices: impl IntoIterator) -> Self { + let (runtime, pending_irqs) = Self::from_rdif_devices_deferred(devices); + if !pending_irqs.is_empty() { + warn!( + "rdif filesystem block devices registered IRQ handlers without an installed drain \ + task; keeping completion mode polling" + ); + } + drop(pending_irqs); + runtime + } + + fn from_rdif_devices_deferred( + devices: impl IntoIterator, + ) -> (Self, Vec) { let mut runtime = Self::new(); + let mut pending_irqs = Vec::new(); for block in devices { let device_index = runtime.devices.len(); let drain_wake = Arc::new(RuntimeDrainWake { device_index }); match build_rdif_block_device(block, device_index, drain_wake) { Ok(registered) => { - let (device, registrations) = registered; - for registration in registrations { - runtime.push_irq_registration(registration); + let (device, pending_irq) = registered; + if let Some(pending_irq) = pending_irq { + pending_irqs.push(pending_irq); } runtime.push_device(device); } Err(err) => warn!("failed to register rdif filesystem block device: {err:?}"), } } - runtime + (runtime, pending_irqs) } pub fn install_from_rdif_devices( devices: impl IntoIterator, ) -> Arc { - let runtime = Arc::new(Self::from_rdif_devices(devices)); + let (runtime, pending_irqs) = Self::from_rdif_devices_deferred(devices); + let runtime = Arc::new(runtime); BLOCK_RUNTIME.call_once(|| runtime.clone()); spawn_block_drain_task(runtime.clone()); + for pending_irq in pending_irqs { + runtime.enable_deferred_rdif_irq(pending_irq); + } runtime } + + fn enable_deferred_rdif_irq(&self, pending: DeferredRdifBlockIrq) { + let DeferredRdifBlockIrq { + name, + block, + device, + registrations, + } = pending; + + block.enable_irq(); + if block.is_irq_enabled() { + device.set_completion_mode(BlockCompletionMode::IrqDriven); + for registration in registrations { + self.push_irq_registration(registration); + } + warn!("rdif filesystem block device {name} registered with IRQ-driven completion"); + } else { + block.disable_irq(); + drop(registrations); + warn!( + "rdif filesystem block device {name} registered IRQ handler but device did not \ + enable completion IRQ; falling back to polling" + ); + } + } } impl BlockDeviceHandle { @@ -494,6 +545,14 @@ impl BlockDeviceHandle { }) } + pub fn drain_pending_requests(&self) -> usize { + self.with_drain(|| { + let keys = self.pending.lock().active_keys(); + let mut poller = DeviceRequestPoller { device: self }; + CompletionDrain::new(&self.pending, &mut poller).poll_keys(&keys) + }) + } + pub fn drain_hint(&self, hint: CompletionHint) -> usize { self.with_drain(|| { let mut poller = DeviceRequestPoller { device: self }; @@ -608,10 +667,11 @@ impl BlockDeviceHandle { let info = queue.info; match submit_flush_request(&mut queue, info) { Ok(request_id) => { + let token = queue.queue.request_token(request_id); let key = self .pending .lock() - .insert_submitted(info.id, request_id, None) + .insert_submitted(info.id, request_id, token, None) .map_err(map_blk_err_to_ax_err)?; drop(queue); self.wake_drain_after_irq_submit(); @@ -939,7 +999,8 @@ impl BlockDeviceHandle { self.poison_driver_contract_violation(); return Err(BlkError::InvalidRequest); } - pending.insert_submitted(info.id, request_id, buffer)? + let token = queue.queue.request_token(request_id); + pending.insert_submitted(info.id, request_id, token, buffer)? }; Ok(WindowEntry { key, @@ -1199,9 +1260,16 @@ impl BlockDeviceHandle { } type BlockIrqRegistrations = Vec>; -type RegisteredRdifBlockDevice = (Arc, BlockIrqRegistrations); +type RegisteredRdifBlockDevice = (Arc, Option); type RegisterIrqResult = Result; +struct DeferredRdifBlockIrq { + name: String, + block: RdifBlockDevice, + device: Arc, + registrations: BlockIrqRegistrations, +} + fn build_rdif_block_device( mut block: RdifBlockDevice, device_index: usize, @@ -1230,19 +1298,7 @@ fn build_rdif_block_device( ) .map_err(map_blk_err_to_ax_err)?; - let registrations = match register_rdif_irq_handlers(&mut block, device.clone(), device_index) - .and_then(|registrations| { - block.enable_irq(); - if block.is_irq_enabled() { - Ok(registrations) - } else { - warn!( - "rdif filesystem block device {name} registered IRQ handler but device did \ - not enable completion IRQ" - ); - Err((AxError::Unsupported, registrations)) - } - }) { + let registrations = match register_rdif_irq_handlers(&mut block, device.clone(), device_index) { Ok(registrations) => registrations, Err((err, registrations)) => { block.disable_irq(); @@ -1251,12 +1307,14 @@ fn build_rdif_block_device( Vec::new() } }; - if !registrations.is_empty() { - device.set_completion_mode(BlockCompletionMode::IrqDriven); - warn!("rdif filesystem block device {name} registered with IRQ-driven completion"); - } + let pending_irq = (!registrations.is_empty()).then_some(DeferredRdifBlockIrq { + name: name.clone(), + block, + device: device.clone(), + registrations, + }); info!("registered rdif filesystem block device {name}"); - Ok((device, registrations)) + Ok((device, pending_irq)) } fn register_rdif_irq_handlers( @@ -1319,14 +1377,23 @@ fn mark_block_drain_device_from_irq(device_index: usize) { notify_drain_from_irq(); } -fn block_drain_has_pending() -> bool { +fn runtime_has_ready_bridge(runtime: &BlockRuntime) -> bool { + runtime + .devices() + .iter() + .any(|device| device.bridge().drain_ready()) +} + +fn block_drain_has_pending(runtime: &BlockRuntime) -> bool { BLOCK_DRAIN_FULL_SCAN.load(Ordering::Acquire) || BLOCK_DRAIN_DEVICE_BITS.load(Ordering::Acquire) != 0 + || runtime_has_ready_bridge(runtime) } -fn take_block_drain_selection() -> DrainSelection { +fn take_block_drain_selection(runtime: &BlockRuntime) -> DrainSelection { DrainSelection { - full_scan: BLOCK_DRAIN_FULL_SCAN.swap(false, Ordering::AcqRel), + full_scan: BLOCK_DRAIN_FULL_SCAN.swap(false, Ordering::AcqRel) + || runtime_has_ready_bridge(runtime), device_bits: BLOCK_DRAIN_DEVICE_BITS.swap(0, Ordering::AcqRel), } } @@ -1336,22 +1403,40 @@ fn drain_selection_contains(selection: DrainSelection, device_index: usize) -> b || (device_index < u64::BITS as usize && selection.device_bits & (1 << device_index) != 0) } +fn drain_selected_device(device_index: usize, device: &BlockDeviceHandle) { + device.drain_events(); + if device.bridge().drain_ready() { + set_block_drain_pending(device_index); + } +} + +fn scan_block_drain_sources(runtime: &BlockRuntime) -> bool { + let mut completed = 0; + for (device_index, device) in runtime.devices().iter().enumerate() { + completed += device.drain_pending_requests(); + if device.bridge().drain_ready() { + set_block_drain_pending(device_index); + } + } + completed != 0 || block_drain_has_pending(runtime) +} + fn spawn_block_drain_task(runtime: Arc) { BLOCK_DRAIN_SPAWNED.call_once(|| { spawn_task( String::from("block_drain"), Box::new(move || { loop { - if !block_drain_has_pending() { + if !block_drain_has_pending(&runtime) && !scan_block_drain_sources(&runtime) { wait_for_drain_notification(); } - if !block_drain_has_pending() { + if !block_drain_has_pending(&runtime) && !scan_block_drain_sources(&runtime) { continue; } - let selection = take_block_drain_selection(); + let selection = take_block_drain_selection(&runtime); for (device_index, device) in runtime.devices().iter().enumerate() { if drain_selection_contains(selection, device_index) { - device.drain_events(); + drain_selected_device(device_index, device); } } } @@ -1566,3 +1651,161 @@ pub fn map_blk_err_to_ax_err(err: BlkError) -> AxError { fn queue_ids_from_bits(bits: u64) -> impl Iterator { (0..u64::BITS as usize).filter(move |queue_id| bits & (1 << queue_id) != 0) } + +#[cfg(test)] +mod tests { + use alloc::sync::Arc; + use core::sync::atomic::Ordering; + use std::sync::Mutex; + + use rdif_block::{QueueLimits, RequestStatus}; + + use super::*; + + static BLOCK_DRAIN_TEST_LOCK: Mutex<()> = Mutex::new(()); + + struct BusyDrainQueue; + + // SAFETY: This test queue never retains request segment references beyond + // `submit_request`; the test does not submit any request through it. + unsafe impl IQueue for BusyDrainQueue { + fn id(&self) -> usize { + 0 + } + + fn info(&self) -> QueueInfo { + QueueInfo { + id: 0, + device: DeviceInfo::new(16, 512), + limits: QueueLimits::simple(512, u64::MAX), + } + } + + fn submit_request(&mut self, _request: Request<'_>) -> Result { + Ok(RequestId::new(1)) + } + + fn poll_request(&mut self, _request: RequestId) -> Result { + Ok(RequestStatus::Pending) + } + } + + struct ReadyWithoutHintQueue; + + // SAFETY: This test queue never retains request segment references beyond + // `submit_request`; the test inserts pending requests directly. + unsafe impl IQueue for ReadyWithoutHintQueue { + fn id(&self) -> usize { + 0 + } + + fn info(&self) -> QueueInfo { + QueueInfo { + id: 0, + device: DeviceInfo::new(16, 512), + limits: QueueLimits::simple(512, u64::MAX), + } + } + + fn submit_request(&mut self, _request: Request<'_>) -> Result { + Ok(RequestId::new(7)) + } + + fn poll_request(&mut self, _request: RequestId) -> Result { + Ok(RequestStatus::Complete) + } + } + + #[test] + fn drain_selection_rearms_device_when_core_drain_is_busy() { + let _guard = BLOCK_DRAIN_TEST_LOCK.lock().unwrap(); + BLOCK_DRAIN_DEVICE_BITS.store(0, Ordering::Release); + BLOCK_DRAIN_FULL_SCAN.store(false, Ordering::Release); + + let bridge = Arc::new(BlockIrqBridge::new()); + let device = BlockDeviceHandle::new( + "busy-drain", + [Box::new(BusyDrainQueue) as Box], + bridge.clone(), + BlockRuntimeConfig::new(Arc::new(NoopDrainWake)), + ) + .unwrap(); + + bridge.record_hint(CompletionHint::Queue { queue_id: 0 }); + set_block_drain_pending(0); + let mut runtime = BlockRuntime::new(); + runtime.push_device(device.clone()); + + let selection = take_block_drain_selection(&runtime); + assert!(drain_selection_contains(selection, 0)); + assert_eq!(BLOCK_DRAIN_DEVICE_BITS.load(Ordering::Acquire), 0); + + device.drain_running.store(true, Ordering::Release); + drain_selected_device(0, &device); + device.drain_running.store(false, Ordering::Release); + + assert!(bridge.drain_ready()); + assert_ne!(BLOCK_DRAIN_DEVICE_BITS.load(Ordering::Acquire) & 1, 0); + + BLOCK_DRAIN_DEVICE_BITS.store(0, Ordering::Release); + BLOCK_DRAIN_FULL_SCAN.store(false, Ordering::Release); + } + + #[test] + fn drain_selection_scans_backend_state_when_global_bit_is_lost() { + let _guard = BLOCK_DRAIN_TEST_LOCK.lock().unwrap(); + BLOCK_DRAIN_DEVICE_BITS.store(0, Ordering::Release); + BLOCK_DRAIN_FULL_SCAN.store(false, Ordering::Release); + + let bridge = Arc::new(BlockIrqBridge::new()); + let device = BlockDeviceHandle::new( + "lost-drain-bit", + [Box::new(BusyDrainQueue) as Box], + bridge.clone(), + BlockRuntimeConfig::new(Arc::new(NoopDrainWake)), + ) + .unwrap(); + let mut runtime = BlockRuntime::new(); + runtime.push_device(device); + + bridge.record_hint(CompletionHint::Queue { queue_id: 0 }); + assert!(block_drain_has_pending(&runtime)); + + let selection = take_block_drain_selection(&runtime); + assert!(selection.full_scan); + assert!(drain_selection_contains(selection, 0)); + + BLOCK_DRAIN_DEVICE_BITS.store(0, Ordering::Release); + BLOCK_DRAIN_FULL_SCAN.store(false, Ordering::Release); + } + + #[test] + fn drain_worker_scans_pending_backend_state_before_sleep() { + let _guard = BLOCK_DRAIN_TEST_LOCK.lock().unwrap(); + BLOCK_DRAIN_DEVICE_BITS.store(0, Ordering::Release); + BLOCK_DRAIN_FULL_SCAN.store(false, Ordering::Release); + + let bridge = Arc::new(BlockIrqBridge::new()); + let device = BlockDeviceHandle::new( + "ready-without-hint", + [Box::new(ReadyWithoutHintQueue) as Box], + bridge, + BlockRuntimeConfig::new(Arc::new(NoopDrainWake)), + ) + .unwrap(); + let key = device + .pending + .lock() + .insert_submitted(0, RequestId::new(7), None, None) + .unwrap(); + let mut runtime = BlockRuntime::new(); + runtime.push_device(device.clone()); + + assert!(!block_drain_has_pending(&runtime)); + assert!(scan_block_drain_sources(&runtime)); + assert!(device.pending.lock().result(key).is_some()); + + BLOCK_DRAIN_DEVICE_BITS.store(0, Ordering::Release); + BLOCK_DRAIN_FULL_SCAN.store(false, Ordering::Release); + } +} diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/irq.rs b/os/arceos/modules/axfs-ng/src/block_runtime/irq.rs index f35be58dc7..87315d1b23 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/irq.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/irq.rs @@ -6,6 +6,7 @@ use rdif_block::{CompletionHint, CompletionList, Event}; const IRQ_HINT_SLOTS: usize = rdif_block::MAX_COMPLETION_HINTS; pub struct BlockIrqBridge { + seq: AtomicU64, queue_bits: AtomicU64, hint_slots: [AtomicHintSlot; IRQ_HINT_SLOTS], drain_ready: AtomicBool, @@ -18,6 +19,7 @@ pub struct RuntimeEventLatch { #[derive(Clone, Copy, Debug)] pub struct DrainEvents { + pub seq: u64, pub queue_bits: u64, pub hints: CompletionList, } @@ -71,6 +73,7 @@ impl RuntimeEventLatch { queue_id, request_id, }, + CompletionHint::Token { token, .. } => CompletionHint::Token { queue_id, token }, CompletionHint::Batch { ids, .. } => CompletionHint::Batch { queue_id, ids }, }) } @@ -79,6 +82,7 @@ impl RuntimeEventLatch { impl BlockIrqBridge { pub const fn new() -> Self { Self { + seq: AtomicU64::new(0), queue_bits: AtomicU64::new(0), hint_slots: [const { AtomicHintSlot::new() }; IRQ_HINT_SLOTS], drain_ready: AtomicBool::new(false), @@ -94,26 +98,43 @@ impl BlockIrqBridge { if !event.completions.is_empty() { for hint in event.completions.iter() { if !self.push_hint_slot(hint) { - self.record_queue_ready(hint.queue_id()); + self.record_queue_ready_inner(hint.queue_id()); } } } - self.drain_ready.store(true, Ordering::Release); + self.publish_seq(); } pub fn record_hint(&self, hint: CompletionHint) { if !self.push_hint_slot(hint) { - self.record_queue_ready(hint.queue_id()); + self.record_queue_ready_inner(hint.queue_id()); } - self.drain_ready.store(true, Ordering::Release); + self.publish_seq(); } pub fn record_queue_ready(&self, queue_id: usize) { + self.record_queue_ready_inner(queue_id); + self.publish_seq(); + } + + fn record_queue_ready_inner(&self, queue_id: usize) { if queue_id < u64::BITS as usize { self.queue_bits.fetch_or(1 << queue_id, Ordering::AcqRel); } + } + + fn publish_seq(&self) { self.drain_ready.store(true, Ordering::Release); + self.seq.fetch_add(1, Ordering::AcqRel); + } + + pub fn seq(&self) -> u64 { + self.seq.load(Ordering::Acquire) + } + + pub fn has_changed(&self, observed: u64) -> bool { + self.seq() != observed } pub fn drain_ready(&self) -> bool { @@ -121,6 +142,7 @@ impl BlockIrqBridge { } pub fn take_events(&self) -> DrainEvents { + let seq = self.seq(); self.drain_ready.store(false, Ordering::Release); let queue_bits = self.queue_bits.swap(0, Ordering::AcqRel); let mut hints = CompletionList::new(); @@ -134,7 +156,11 @@ impl BlockIrqBridge { { self.drain_ready.store(true, Ordering::Release); } - DrainEvents { queue_bits, hints } + DrainEvents { + seq, + queue_bits, + hints, + } } fn push_hint_slot(&self, hint: CompletionHint) -> bool { @@ -158,6 +184,7 @@ struct AtomicHintSlot { kind: AtomicUsize, queue_id: AtomicUsize, request_id: AtomicUsize, + request_generation: AtomicU64, batch_len: AtomicUsize, batch_ids: [AtomicUsize; rdif_block::MAX_BATCH_COMPLETION_IDS], } @@ -173,6 +200,7 @@ impl AtomicHintSlot { kind: AtomicUsize::new(0), queue_id: AtomicUsize::new(0), request_id: AtomicUsize::new(0), + request_generation: AtomicU64::new(0), batch_len: AtomicUsize::new(0), batch_ids: [const { AtomicUsize::new(0) }; rdif_block::MAX_BATCH_COMPLETION_IDS], } @@ -202,6 +230,15 @@ impl AtomicHintSlot { self.kind.store(1, Ordering::Relaxed); self.request_id .store(usize::from(request_id), Ordering::Relaxed); + self.request_generation.store(0, Ordering::Relaxed); + self.batch_len.store(0, Ordering::Relaxed); + } + CompletionHint::Token { token, .. } => { + self.kind.store(3, Ordering::Relaxed); + self.request_id + .store(usize::from(token.id), Ordering::Relaxed); + self.request_generation + .store(token.generation.get(), Ordering::Relaxed); self.batch_len.store(0, Ordering::Relaxed); } CompletionHint::Batch { ids, .. } => { @@ -238,6 +275,15 @@ impl AtomicHintSlot { queue_id, request_id: rdif_block::RequestId::new(self.request_id.load(Ordering::Relaxed)), }, + 3 => CompletionHint::Token { + queue_id, + token: rdif_block::RequestToken::new( + rdif_block::RequestId::new(self.request_id.load(Ordering::Relaxed)), + rdif_block::RequestGeneration::new( + self.request_generation.load(Ordering::Relaxed), + ), + ), + }, 2 => { let mut ids = rdif_block::CompletionIds::new(); let len = self.batch_len.load(Ordering::Relaxed); @@ -294,5 +340,68 @@ mod tests { let events = bridge.take_events(); assert_eq!(events.queue_bits, 1); + assert_eq!(events.seq, 1); + } + + #[test] + fn block_irq_bridge_sequences_coalesced_events() { + let bridge = BlockIrqBridge::new(); + assert_eq!(bridge.seq(), 0); + assert!(!bridge.has_changed(0)); + + bridge.record_queue_ready(0); + bridge.record_queue_ready(1); + + assert!(bridge.has_changed(0)); + assert_eq!(bridge.seq(), 2); + + let events = bridge.take_events(); + assert_eq!(events.seq, 2); + assert_eq!(events.queue_bits, 0b11); + assert!(!bridge.drain_ready()); + } + + #[test] + fn block_irq_bridge_hint_overflow_falls_back_to_queue_scan_once() { + let bridge = BlockIrqBridge::new(); + let mut event = Event::none(); + for id in 0..(IRQ_HINT_SLOTS + 1) { + event.push_request(3, rdif_block::RequestId::new(id)); + } + + bridge.record_event(event); + + assert_eq!(bridge.seq(), 1); + let events = bridge.take_events(); + assert_eq!(events.seq, 1); + assert_eq!(events.hints.len(), IRQ_HINT_SLOTS); + assert_eq!(events.queue_bits, 1 << 3); + } + + #[test] + fn block_irq_bridge_preserves_request_token_hints() { + let bridge = BlockIrqBridge::new(); + let token = rdif_block::RequestToken::new( + rdif_block::RequestId::new(9), + rdif_block::RequestGeneration::new(42), + ); + let mut event = Event::none(); + event.push_token(2, token); + + bridge.record_event(event); + + let events = bridge.take_events(); + let hint = events + .hints + .iter() + .next() + .expect("token hint should be preserved"); + assert!(matches!( + hint, + CompletionHint::Token { + queue_id: 2, + token: observed, + } if observed == token + )); } } diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/mod.rs b/os/arceos/modules/axfs-ng/src/block_runtime/mod.rs index 0d1adc290a..06b688c5cb 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/mod.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/mod.rs @@ -2,11 +2,11 @@ pub use crate::block::runtime::*; #[cfg(test)] mod tests { - use alloc::{boxed::Box, collections::BTreeMap, sync::Arc, vec::Vec}; + use alloc::{boxed::Box, collections::BTreeMap, string::String, sync::Arc, vec::Vec}; use core::{ any::Any, cell::Cell, - sync::atomic::{AtomicU64, AtomicUsize, Ordering}, + sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, }; use std::{ collections::HashMap, @@ -15,17 +15,24 @@ mod tests { use ax_errno::AxError; use dma_api::{DeviceDma, DmaDomainId}; + use irq_framework::{HwIrq, IrqContext, IrqDomainId, IrqId}; use rdif_block::{ - BlkError, CompletionHint, DeviceInfo, DriverGeneric, IQueue, IQueueOwned, Interface, - OwnedRequest, PollError, QueueHandle, QueueInfo, QueueLimits, Request, RequestId, - RequestOp, RequestPoll, RequestStatus, SubmitError, + BlkError, CompletionHint, DeviceInfo, DriverGeneric, Event as RdifIrqEvent, IQueue, + IQueueOwned, IdList, Interface, IrqHandler, IrqSourceInfo, OwnedRequest, PollError, + QueueHandle, QueueInfo, QueueLimits, Request, RequestId, RequestOp, RequestPoll, + RequestStatus, SubmitError, }; use super::*; - use crate::os::{BlockTaskOps, install_dma_op, set_task_ops, sync::IrqMutex as SpinNoIrq}; + use crate::os::{ + BlockIrqOutcome, BlockIrqRegistrar, BlockIrqRegistration, BlockTaskOps, install_dma_op, + set_irq_registrar, set_task_ops, sync::IrqMutex as SpinNoIrq, + }; static TEST_TASK_OPS: TestTaskOps = TestTaskOps; + static TEST_IRQ_REGISTRAR: TestIrqRegistrar = TestIrqRegistrar; static NEXT_TEST_TASK_ID: AtomicU64 = AtomicU64::new(1_000_000); + static TEST_SPAWNS: AtomicUsize = AtomicUsize::new(0); static TEST_TIMEOUT_WAITS: AtomicUsize = AtomicUsize::new(0); static TEST_TASKS: OnceLock>>> = OnceLock::new(); static TEST_TASK_LOCK: StdMutex<()> = StdMutex::new(()); @@ -137,6 +144,49 @@ mod tests { fn wait_for_drain_notification(&self) { self.task_wait(); } + + fn spawn(&self, _name: String, _f: Box) { + TEST_SPAWNS.fetch_add(1, Ordering::AcqRel); + } + } + + struct TestIrqRegistrar; + + struct TestIrqRegistration; + + impl BlockIrqRegistration for TestIrqRegistration {} + + impl BlockIrqRegistrar for TestIrqRegistrar { + fn register_shared( + &self, + _name: String, + _irq: IrqId, + _action: Box BlockIrqOutcome + Send + 'static>, + ) -> Result, AxError> { + Ok(Box::new(TestIrqRegistration)) + } + } + + struct IrqEnableTrace { + enabled: AtomicBool, + enable_seen_spawns: AtomicUsize, + } + + impl IrqEnableTrace { + fn new() -> Self { + Self { + enabled: AtomicBool::new(false), + enable_seen_spawns: AtomicUsize::new(0), + } + } + } + + struct TestIrqHandler; + + impl IrqHandler for TestIrqHandler { + fn handle_irq(&mut self) -> RdifIrqEvent { + RdifIrqEvent::from_queue_bits(1) + } } fn install_test_task_ops() { @@ -383,7 +433,9 @@ mod tests { #[test] fn request_completes_before_wait_token_registration() { let mut table = PendingTable::new(); - table.insert_submitted(0, RequestId::new(1), None).unwrap(); + table + .insert_submitted(0, RequestId::new(1), None, None) + .unwrap(); assert!(table.complete(key(1), Ok(())).is_none()); assert_eq!(table.register_waiter_task(key(1), 7), Some(Ok(()))); @@ -396,7 +448,9 @@ mod tests { #[test] fn request_completes_after_waiter_task_registration() { let mut table = PendingTable::new(); - let key = table.insert_submitted(0, RequestId::new(2), None).unwrap(); + let key = table + .insert_submitted(0, RequestId::new(2), None, None) + .unwrap(); assert_eq!(table.register_waiter_task(key, 7), None); let wake = table.complete(key, Ok(())).unwrap(); @@ -410,9 +464,13 @@ mod tests { #[test] fn runtime_request_key_survives_driver_request_id_reuse() { let mut table = PendingTable::new(); - let first = table.insert_submitted(0, RequestId::new(1), None).unwrap(); + let first = table + .insert_submitted(0, RequestId::new(1), None, None) + .unwrap(); assert!(table.complete(first, Ok(())).is_none()); - let second = table.insert_submitted(0, RequestId::new(1), None).unwrap(); + let second = table + .insert_submitted(0, RequestId::new(1), None, None) + .unwrap(); assert_ne!(first, second); assert_eq!( @@ -426,10 +484,12 @@ mod tests { #[test] fn pending_table_rejects_inflight_driver_request_id_reuse() { let mut table = PendingTable::new(); - table.insert_submitted(0, RequestId::new(1), None).unwrap(); + table + .insert_submitted(0, RequestId::new(1), None, None) + .unwrap(); assert_eq!( - table.insert_submitted(0, RequestId::new(1), None), + table.insert_submitted(0, RequestId::new(1), None, None), Err(BlkError::InvalidRequest) ); } @@ -439,11 +499,11 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); pending .lock() - .insert_submitted(0, RequestId::new(2), None) + .insert_submitted(0, RequestId::new(2), None, None) .unwrap(); pending.lock().register_waiter_task(key(1), 1); pending.lock().register_waiter_task(key(2), 2); @@ -459,16 +519,55 @@ mod tests { assert!(pending.lock().request(key(2)).is_some()); } + #[test] + fn token_hint_wakes_only_matching_generation() { + let pending = SpinNoIrq::new(PendingTable::new()); + let stale = + rdif_block::RequestToken::new(RequestId::new(1), rdif_block::RequestGeneration::new(1)); + let current = + rdif_block::RequestToken::new(RequestId::new(1), rdif_block::RequestGeneration::new(2)); + let key = pending + .lock() + .insert_submitted(0, RequestId::new(1), Some(current), None) + .unwrap(); + pending.lock().register_waiter_task(key, 1); + + let mut poller = Poller::default(); + poller.complete(driver_key(1)); + let mut drain = CompletionDrain::new(&pending, &mut poller); + + assert_eq!( + drain.drain_hint(CompletionHint::Token { + queue_id: 0, + token: stale, + }), + 0 + ); + assert_eq!(pending.lock().result(key), None); + + assert_eq!( + drain.drain_hint(CompletionHint::Token { + queue_id: 0, + token: current, + }), + 1 + ); + assert_eq!( + pending.lock().take_completed(key).map(|(result, _)| result), + Some(Ok(())) + ); + } + #[test] fn queue_hint_scans_all_pending_requests_on_queue() { let pending = SpinNoIrq::new(PendingTable::new()); pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); pending .lock() - .insert_submitted(0, RequestId::new(2), None) + .insert_submitted(0, RequestId::new(2), None, None) .unwrap(); pending.lock().register_waiter_task(key(1), 1); pending.lock().register_waiter_task(key(2), 2); @@ -485,11 +584,11 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); pending .lock() - .insert_submitted(0, RequestId::new(2), None) + .insert_submitted(0, RequestId::new(2), None, None) .unwrap(); pending.lock().register_waiter_task(key(1), 1); pending.lock().register_waiter_task(key(2), 2); @@ -511,15 +610,15 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); pending .lock() - .insert_submitted(0, RequestId::new(2), None) + .insert_submitted(0, RequestId::new(2), None, None) .unwrap(); pending .lock() - .insert_submitted(0, RequestId::new(3), None) + .insert_submitted(0, RequestId::new(3), None, None) .unwrap(); pending.lock().register_waiter_task(key(1), 1); pending.lock().register_waiter_task(key(2), 2); @@ -548,7 +647,7 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); let key = pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); pending.lock().register_waiter_task(key, 4); @@ -572,11 +671,11 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); pending .lock() - .insert_submitted(1, RequestId::new(7), None) + .insert_submitted(1, RequestId::new(7), None, None) .unwrap(); pending.lock().register_waiter_task(key(1), 1); pending.lock().register_waiter_task(key(2), 2); @@ -589,6 +688,7 @@ mod tests { assert_eq!( drain.drain_events(DrainEvents { + seq: 1, queue_bits: 0b11, hints: rdif_block::CompletionList::new(), }), @@ -603,7 +703,7 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); assert!(pending.lock().complete(key(1), Ok(())).is_none()); @@ -627,7 +727,7 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); assert!(pending.lock().complete(key(1), Ok(())).is_none()); @@ -655,7 +755,9 @@ mod tests { #[test] fn pending_table_allows_only_one_active_poll_per_request() { let mut table = PendingTable::new(); - table.insert_submitted(0, RequestId::new(1), None).unwrap(); + table + .insert_submitted(0, RequestId::new(1), None, None) + .unwrap(); assert_eq!(table.begin_poll(key(1)), PollClaim::Claimed); assert_eq!(table.begin_poll(key(1)), PollClaim::AlreadyPolling); @@ -667,7 +769,9 @@ mod tests { #[test] fn completed_request_cannot_reenter_poll_after_submit_side_completion() { let mut table = PendingTable::new(); - table.insert_submitted(0, RequestId::new(1), None).unwrap(); + table + .insert_submitted(0, RequestId::new(1), None, None) + .unwrap(); assert_eq!(table.begin_poll(key(1)), PollClaim::Claimed); assert!(table.complete(key(1), Ok(())).is_none()); @@ -715,7 +819,7 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); let key = pending .lock() - .insert_submitted(0, RequestId::new(1), None) + .insert_submitted(0, RequestId::new(1), None, None) .unwrap(); pending.lock().register_waiter_task(key, 1); @@ -745,7 +849,7 @@ mod tests { let pending = SpinNoIrq::new(PendingTable::new()); let key = pending .lock() - .insert_submitted(0, RequestId::new(3), None) + .insert_submitted(0, RequestId::new(3), None, None) .unwrap(); pending.lock().register_waiter_task(key, 1); @@ -784,7 +888,12 @@ mod tests { .unwrap(); let key = pending .lock() - .insert_submitted(0, RequestId::new(4), Some(RuntimeDmaBuffer::Legacy(guard))) + .insert_submitted( + 0, + RequestId::new(4), + None, + Some(RuntimeDmaBuffer::Legacy(guard)), + ) .unwrap(); pending.lock().abandon(key); assert_eq!( @@ -1135,6 +1244,68 @@ mod tests { } } + struct IrqMockInterface { + name: &'static str, + queue: Option>, + info: QueueInfo, + trace: Arc, + } + + impl IrqMockInterface { + fn new(queue: MockQueue, trace: Arc) -> Self { + let info = queue.info(); + Self { + name: "irq-mock-rdif", + queue: Some(Box::new(queue)), + info, + trace, + } + } + } + + impl DriverGeneric for IrqMockInterface { + fn name(&self) -> &str { + self.name + } + } + + impl Interface for IrqMockInterface { + fn device_info(&self) -> DeviceInfo { + self.info.device + } + + fn queue_limits(&self) -> QueueLimits { + self.info.limits + } + + fn create_queue(&mut self) -> Option> { + self.queue.take() + } + + fn enable_irq(&self) { + self.trace + .enable_seen_spawns + .store(TEST_SPAWNS.load(Ordering::Acquire), Ordering::Release); + self.trace.enabled.store(true, Ordering::Release); + } + + fn disable_irq(&self) { + self.trace.enabled.store(false, Ordering::Release); + } + + fn is_irq_enabled(&self) -> bool { + self.trace.enabled.load(Ordering::Acquire) + } + + fn irq_sources(&self) -> Vec { + vec![IrqSourceInfo::legacy(IdList::from_bits(1))] + } + + fn take_irq_handler(&mut self, source_id: usize) -> Option> { + (source_id == 0).then(|| Box::new(TestIrqHandler) as Box) + } + } + struct MockOwnedQueue { info: QueueInfo, next: usize, @@ -1286,6 +1457,33 @@ mod tests { assert_eq!(buf[0], 7); } + #[test] + fn install_enables_irq_after_drain_task_is_spawned() { + let _guard = test_task_guard(); + install_test_task_ops(); + set_irq_registrar(&TEST_IRQ_REGISTRAR); + TEST_SPAWNS.store(0, Ordering::Release); + let trace = Arc::new(IrqEnableTrace::new()); + + let runtime = BlockRuntime::install_from_rdif_devices([RdifBlockDevice::new( + "irq-mock-rdif", + Some(IrqId::new(IrqDomainId(0), HwIrq(32))), + Box::new(IrqMockInterface::new(MockQueue::new(), trace.clone())), + )]); + + assert_eq!(runtime.devices().len(), 1); + assert!(trace.enabled.load(Ordering::Acquire)); + assert_eq!( + trace.enable_seen_spawns.load(Ordering::Acquire), + 1, + "IRQ-driven mode must not be enabled before the block drain task is spawned", + ); + assert_eq!( + runtime.devices()[0].completion_mode(), + BlockCompletionMode::IrqDriven, + ); + } + #[test] fn runtime_prefers_owned_queue_from_rdif_interface() { let _guard = test_task_guard(); diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/queue.rs b/os/arceos/modules/axfs-ng/src/block_runtime/queue.rs index 63f2ef815d..b89ba44150 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/queue.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/queue.rs @@ -1,4 +1,4 @@ -use rdif_block::{BlkError, CompletionHint, RequestId}; +use rdif_block::{BlkError, CompletionHint, RequestId, RequestToken}; use super::{DrainEvents, PendingTable, PollClaim, PollProgress, RequestKey, RuntimeDmaBuffer}; use crate::os::{sync::IrqMutex as SpinNoIrq, wake_task}; @@ -115,6 +115,9 @@ impl<'a, P: RequestPoller> CompletionDrain<'a, P> { queue_id, request_id, } => self.poll_matching_driver_requests(queue_id, &[request_id]), + CompletionHint::Token { queue_id, token } => { + self.poll_matching_driver_token(queue_id, token) + } CompletionHint::Batch { queue_id, ids } => { let ids = ids.iter().collect::>(); self.poll_matching_driver_requests(queue_id, &ids) @@ -151,6 +154,14 @@ impl<'a, P: RequestPoller> CompletionDrain<'a, P> { self.pending.lock().matching_driver_keys(queue_id, ids) } + fn poll_matching_driver_token(&mut self, queue_id: usize, token: RequestToken) -> usize { + let keys = self + .pending + .lock() + .matching_driver_token_keys(queue_id, token); + self.poll_batch(&keys) + } + fn poll_batch(&mut self, keys: &[RequestKey]) -> usize { let batches = self.claim_poll_batches(keys); if batches.is_empty() { diff --git a/os/arceos/modules/axfs-ng/src/block_runtime/request.rs b/os/arceos/modules/axfs-ng/src/block_runtime/request.rs index 95d61c4e16..43bcb2676f 100644 --- a/os/arceos/modules/axfs-ng/src/block_runtime/request.rs +++ b/os/arceos/modules/axfs-ng/src/block_runtime/request.rs @@ -1,6 +1,6 @@ use alloc::{collections::BTreeMap, vec::Vec}; -use rdif_block::{BlkError, RequestId}; +use rdif_block::{BlkError, RequestId, RequestToken}; use super::RuntimeDmaBuffer; @@ -25,6 +25,7 @@ pub type RequestKey = RuntimeRequestId; pub struct SubmittedRequest { pub queue_id: usize, pub request_id: RequestId, + pub token: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -187,6 +188,7 @@ impl PendingTable { &mut self, queue_id: usize, request_id: RequestId, + token: Option, buffer_guard: Option, ) -> Result { if self.contains_inflight_driver_request(queue_id, request_id) { @@ -200,6 +202,7 @@ impl PendingTable { SubmittedRequest { queue_id, request_id, + token, }, buffer_guard, ), @@ -329,6 +332,22 @@ impl PendingTable { .collect() } + pub fn matching_driver_token_keys( + &self, + queue_id: usize, + token: RequestToken, + ) -> Vec { + self.requests + .iter() + .filter_map(|(key, request)| { + (request.submitted.queue_id == queue_id + && request.result.is_none() + && request.submitted.token == Some(token)) + .then_some(*key) + }) + .collect() + } + pub fn pending_queue_bits(&self) -> u64 { let mut bits = 0u64; for request in self.requests.values() { diff --git a/os/arceos/modules/axhal/Cargo.toml b/os/arceos/modules/axhal/Cargo.toml index 46c8b05d69..6eaba85722 100644 --- a/os/arceos/modules/axhal/Cargo.toml +++ b/os/arceos/modules/axhal/Cargo.toml @@ -12,11 +12,6 @@ smp = [ "axplat-dyn/smp", "ax-plat/smp", ] -irq = [ - "ax-plat/irq", - "axplat-dyn/irq", - "dep:ax-kspin", -] fp-simd = [ "ax-cpu/fp-simd", "axplat-dyn/fp-simd", @@ -42,12 +37,13 @@ axvisor-linker = [] host-test = [] default = [] -ipi = ["irq"] +ipi = [] [dependencies] ax-alloc = { workspace = true, optional = true } ax-cpu.workspace = true -ax-kspin = { workspace = true, optional = true } +ax-crate-interface = { workspace = true, features = ["weak_default"] } +ax-kspin = { workspace = true } ax-plat.workspace = true cfg-if.workspace = true fdt-parser = "0.4" diff --git a/os/arceos/modules/axhal/src/dummy.rs b/os/arceos/modules/axhal/src/dummy.rs index bbeb745c85..230660e743 100644 --- a/os/arceos/modules/axhal/src/dummy.rs +++ b/os/arceos/modules/axhal/src/dummy.rs @@ -1,11 +1,10 @@ //! Dummy implementation of platform-related interfaces defined in [`axplat`]. -#[cfg(feature = "irq")] -use ax_plat::irq::{HwIrq, IpiTarget, IrqError, IrqId, IrqIf, IrqNumber, IrqSource, TrapVector}; use ax_plat::{ console::{ConsoleDeviceIdError, ConsoleDeviceIdResult, ConsoleIf}, impl_plat_interface, init::InitIf, + irq::{HwIrq, IpiTarget, IrqError, IrqId, IrqIf, IrqNumber, IrqSource, TrapVector}, mem::{MemIf, RawRange}, power::PowerIf, time::TimeIf, @@ -16,7 +15,6 @@ struct DummyConsole; struct DummyMem; struct DummyTime; struct DummyPower; -#[cfg(feature = "irq")] struct DummyIrq; #[impl_plat_interface] @@ -48,15 +46,12 @@ impl ConsoleIf for DummyConsole { fn claim_runtime_output() {} - #[cfg(feature = "irq")] fn irq_num() -> Option { None } - #[cfg(feature = "irq")] fn set_input_irq_enabled(_enabled: bool) {} - #[cfg(feature = "irq")] fn handle_irq() -> ax_plat::console::ConsoleIrqEvent { ax_plat::console::ConsoleIrqEvent::empty() } @@ -107,12 +102,10 @@ impl TimeIf for DummyTime { 0 } - #[cfg(feature = "irq")] fn irq_num() -> IrqId { IrqNumber(0).expect("dummy legacy IRQ exceeds legacy IRQ width") } - #[cfg(feature = "irq")] fn set_oneshot_timer(_deadline_ns: u64) {} } @@ -134,7 +127,6 @@ impl PowerIf for DummyPower { } } -#[cfg(feature = "irq")] #[impl_plat_interface] impl IrqIf for DummyIrq { fn set_enable(_irq: IrqId, _enabled: bool) -> Result<(), IrqError> { diff --git a/os/arceos/modules/axhal/src/irq.rs b/os/arceos/modules/axhal/src/irq.rs index bbb6753fcb..51a8ff3926 100644 --- a/os/arceos/modules/axhal/src/irq.rs +++ b/os/arceos/modules/axhal/src/irq.rs @@ -15,6 +15,17 @@ pub use ax_plat::irq::{ #[cfg(feature = "ipi")] pub use ax_plat::irq::{IpiTarget, send_ipi}; +/// Optional IRQ epilogue hook used by task runtimes to consume IRQ-safe wake +/// queues before preemption is re-enabled. +#[ax_crate_interface::def_interface] +pub trait IrqEpilogueIf { + /// Runs after the platform IRQ dispatcher returns, still under + /// `NoPreempt`, before normal scheduling is allowed again. + fn drain_irq_wake_queue_current_cpu() -> usize { + 0 + } +} + /// Returns the platform IRQ id used for inter-processor interrupts. #[cfg(feature = "ipi")] pub fn ipi_irq() -> IrqId { @@ -29,6 +40,7 @@ pub fn ipi_irq() -> IrqId { pub fn handle_irq(vector: usize) -> bool { let guard = ax_kernel_guard::NoPreempt::new(); let handled = handle(TrapVector(vector)).is_some(); + let _ = ax_crate_interface::call_interface!(IrqEpilogueIf::drain_irq_wake_queue_current_cpu); drop(guard); // rescheduling may occur when preemption is re-enabled. handled diff --git a/os/arceos/modules/axhal/src/lib.rs b/os/arceos/modules/axhal/src/lib.rs index 1b6385e909..342967972a 100644 --- a/os/arceos/modules/axhal/src/lib.rs +++ b/os/arceos/modules/axhal/src/lib.rs @@ -16,7 +16,6 @@ //! - `smp`: Enable SMP (symmetric multiprocessing) support. //! - `fp-simd`: Enable floating-point and SIMD support. //! - `paging`: Enable page table manipulation. -//! - `irq`: Enable interrupt handling support. //! - `tls`: Enable kernel space thread-local storage support. //! - `rtc`: Enable real-time clock support. //! - `uspace`: Enable user space support. @@ -25,6 +24,7 @@ //! [cargo test]: https://doc.rust-lang.org/cargo/guide/tests.html #![no_std] +#![feature(linkage)] #[allow(unused_imports)] #[macro_use] @@ -53,7 +53,6 @@ pub mod time; #[cfg(feature = "tls")] pub mod tls; -#[cfg(feature = "irq")] pub mod irq; #[cfg(feature = "paging")] @@ -62,11 +61,10 @@ pub mod paging; /// Console input and output. pub mod console { pub use ax_plat::console::{ - ConsoleDeviceId, ConsoleDeviceIdError, ConsoleDeviceIdResult, claim_runtime_output, - device_id, read_bytes, write_bytes, write_text_bytes, + ConsoleDeviceId, ConsoleDeviceIdError, ConsoleDeviceIdResult, ConsoleIrqEvent, + claim_runtime_output, device_id, handle_irq, irq_num, read_bytes, set_input_irq_enabled, + write_bytes, write_text_bytes, }; - #[cfg(feature = "irq")] - pub use ax_plat::console::{ConsoleIrqEvent, handle_irq, irq_num, set_input_irq_enabled}; } /// CPU power management. diff --git a/os/arceos/modules/axhal/src/pmu.rs b/os/arceos/modules/axhal/src/pmu.rs index 2b2959be2d..ad64d8833e 100644 --- a/os/arceos/modules/axhal/src/pmu.rs +++ b/os/arceos/modules/axhal/src/pmu.rs @@ -36,7 +36,6 @@ pub fn cpu_id_raw() -> Option { } /// Returns the platform IRQ id used for PMU overflows. -#[cfg(feature = "irq")] pub fn irq() -> Result { #[cfg(target_arch = "aarch64")] { diff --git a/os/arceos/modules/axhal/src/time.rs b/os/arceos/modules/axhal/src/time.rs index 210a56dfec..54b8ccdc89 100644 --- a/os/arceos/modules/axhal/src/time.rs +++ b/os/arceos/modules/axhal/src/time.rs @@ -2,13 +2,11 @@ pub use ax_plat::time::{ Duration, MICROS_PER_SEC, MILLIS_PER_SEC, NANOS_PER_MICROS, NANOS_PER_MILLIS, NANOS_PER_SEC, - TimeValue, busy_wait, busy_wait_until, current_ticks, epochoffset_nanos, monotonic_time, - monotonic_time_nanos, nanos_to_ticks, ticks_to_nanos, wall_time, wall_time_nanos, + TimeValue, busy_wait, busy_wait_until, current_ticks, epochoffset_nanos, irq_num, + monotonic_time, monotonic_time_nanos, nanos_to_ticks, set_oneshot_timer, ticks_to_nanos, + wall_time, wall_time_nanos, }; -#[cfg(feature = "irq")] -pub use ax_plat::time::{irq_num, set_oneshot_timer}; -#[cfg(feature = "irq")] pub fn enable_timer_irq() { #[cfg(any(test, feature = "host-test"))] {} diff --git a/os/arceos/modules/axruntime/Cargo.toml b/os/arceos/modules/axruntime/Cargo.toml index 39a644bf8f..ac37197401 100644 --- a/os/arceos/modules/axruntime/Cargo.toml +++ b/os/arceos/modules/axruntime/Cargo.toml @@ -17,18 +17,9 @@ ipi = ["dep:ax-ipi"] # Register a lightweight IPI IRQ handler that only acknowledges SGIs so idle # CPUs can wake and re-enter the scheduler. This intentionally does not enable # the ax-ipi callback queue. -wake-ipi = ["irq", "ax-hal/ipi"] -irq = [ - "ax-hal/irq", - "ax-task?/irq", - "dep:ax-kspin", - "dep:ax-percpu", - "dep:axklib", - "dep:rdif-intc", - "ax-driver/irq", -] +wake-ipi = ["ax-hal/ipi", "ax-task?/irq-wake-ipi"] multitask = ["ax-task/multitask"] -paging = ["ax-hal/paging", "dep:ax-mm", "dep:axklib"] +paging = ["ax-hal/paging", "dep:ax-mm"] rtc = ["ax-hal/rtc", "ax-driver/rtc"] smp = ["ax-hal/smp", "ax-task?/smp"] stack-guard-page = ["multitask", "paging", "ax-task/stack-guard-page"] @@ -37,7 +28,6 @@ tls = ["ax-hal/tls", "ax-task?/tls"] display = ["dep:ax-display", "ax-driver/display"] fs = [ - "irq", "multitask", "dma", "dep:ax-errno", @@ -45,7 +35,6 @@ fs = [ "ax-fs-ng/vfs", "dep:axfs-ng-vfs", "dep:spin", - "dep:axklib", "dep:dma-api", "dep:rdif-block", "ax-driver/block", @@ -53,7 +42,7 @@ fs = [ fs-ext4 = ["fs", "ax-fs-ng/ext4"] fs-fat = ["fs", "ax-fs-ng/fat"] input = ["dep:ax-input", "ax-driver/input"] -net = ["dep:ax-net", "dep:rd-net", "dep:spin", "dep:axklib", "ax-driver/net"] +net = ["dep:ax-net", "dep:rd-net", "dep:spin", "ax-driver/net"] # SG2002 AIC8800 SoftAP Wi-Fi. Pulls the FDT probe into ax-driver and installs # the ArceOS runtime glue (the driver cores are OS-independent) before probing. aic8800-wifi = [ @@ -79,17 +68,17 @@ axfs-ng-vfs = {workspace = true, optional = true} ax-hal = {workspace = true} ax-input = {workspace = true, optional = true} ax-ipi = {workspace = true, optional = true} -ax-kspin = {workspace = true, optional = true} +ax-kspin = {workspace = true} ax-lazyinit = {workspace = true} ax-log = {workspace = true} ax-memory-addr = {workspace = true} ax-mm = {workspace = true, optional = true} ax-net = {workspace = true, optional = true} -ax-percpu = {workspace = true, optional = true} +ax-percpu = {workspace = true} ax-plat = {workspace = true} ax-task = {workspace = true, optional = true} axbacktrace = {workspace = true} -axklib = {workspace = true, optional = true} +axklib = {workspace = true} axpanic = {workspace = true} cfg-if = {workspace = true} chrono = {version = "0.4", default-features = false} @@ -97,7 +86,7 @@ dma-api = { workspace = true, optional = true } indoc = "2" irq-framework = { workspace = true } rdif-block = { workspace = true, optional = true } -rdif-intc = { workspace = true, optional = true } +rdif-intc.workspace = true rd-net = { workspace = true, optional = true } rdrive.workspace = true spin = {workspace = true, optional = true} diff --git a/os/arceos/modules/axruntime/build.rs b/os/arceos/modules/axruntime/build.rs index f51f0f3453..d67383dc0f 100644 --- a/os/arceos/modules/axruntime/build.rs +++ b/os/arceos/modules/axruntime/build.rs @@ -122,7 +122,6 @@ fn build_info_source_from(arch: &str, target: &str, mode: &str, config: RuntimeC #[cfg(feature = "fs")] pub const TASK_STACK_SIZE: usize = #task_stack_size; - #[cfg(feature = "irq")] pub const TICKS_PER_SEC: usize = #ticks_per_sec; } .to_string() @@ -223,7 +222,6 @@ mod tests { "pub const CPU_CAPACITY: usize = 16usize;\n", "#[cfg(feature = \"fs\")]\n", "pub const TASK_STACK_SIZE: usize = 262144usize;\n", - "#[cfg(feature = \"irq\")]\n", "pub const TICKS_PER_SEC: usize = 100usize;\n", )) ); diff --git a/os/arceos/modules/axruntime/src/devices.rs b/os/arceos/modules/axruntime/src/devices.rs index 165d250658..7c5c1de0dc 100644 --- a/os/arceos/modules/axruntime/src/devices.rs +++ b/os/arceos/modules/axruntime/src/devices.rs @@ -33,7 +33,7 @@ fn adapt_display_device( ax_display::ErasedDisplayDevice::new(display) } -#[cfg(all(feature = "display", feature = "irq"))] +#[cfg(feature = "display")] fn resolve_display_irq( _name: &str, irq: Option, @@ -41,17 +41,6 @@ fn resolve_display_irq( irq.map(crate::irq::resolve_binding_irq).transpose() } -#[cfg(all(feature = "display", not(feature = "irq")))] -fn resolve_display_irq( - name: &str, - irq: Option, -) -> Result, core::convert::Infallible> { - if irq.is_some() { - warn!("display device {name} has an IRQ binding but IRQ support is disabled"); - } - Ok(None) -} - #[cfg(feature = "input")] pub(crate) fn init_input() { if !rdrive::is_initialized() { @@ -76,7 +65,7 @@ fn adapt_input_device(taken: ax_driver::input::TakenInputDevice) -> ax_input::Er )) } -#[cfg(all(feature = "input", feature = "irq"))] +#[cfg(feature = "input")] fn resolve_input_irq( _name: &str, irq: Option, @@ -84,20 +73,8 @@ fn resolve_input_irq( irq.map(crate::irq::resolve_binding_irq).transpose() } -#[cfg(all(feature = "input", not(feature = "irq")))] -fn resolve_input_irq( - name: &str, - irq: Option, -) -> Result, core::convert::Infallible> { - if irq.is_some() { - warn!("input device {name} has an IRQ binding but IRQ support is disabled"); - } - Ok(None) -} - #[cfg(feature = "net")] pub(crate) fn init_net() { - #[cfg(feature = "irq")] ax_net::set_ethernet_irq_registrar(&crate::irq::NET_IRQ_REGISTRAR); register_unix_namespace(); let config = parse_network_config(); @@ -185,7 +162,7 @@ fn adapt_net_device( } } -#[cfg(all(feature = "net", feature = "irq"))] +#[cfg(feature = "net")] fn resolve_net_irq(name: &str, irq: Option) -> Option { let irq = irq?; match crate::irq::resolve_binding_irq(irq) { @@ -197,14 +174,6 @@ fn resolve_net_irq(name: &str, irq: Option) -> Option, -) -> Option { - None -} - /// Registers wireless devices that carry a link policy with the /// already-initialized network stack (static IP + optional DHCP server + /// dedicated out-of-band RX poll). Plain NICs are handled by `init_network`. diff --git a/os/arceos/modules/axruntime/src/fs/block.rs b/os/arceos/modules/axruntime/src/fs/block.rs index b1a5e7d294..1d6237b8d2 100644 --- a/os/arceos/modules/axruntime/src/fs/block.rs +++ b/os/arceos/modules/axruntime/src/fs/block.rs @@ -1,4 +1,5 @@ use alloc::{boxed::Box, string::String, vec::Vec}; +use core::sync::atomic::{AtomicBool, Ordering}; use ax_alloc::UsageKind; use ax_fs_ng::{ @@ -40,8 +41,9 @@ static TIME_PROVIDER: RuntimeTimeProvider = RuntimeTimeProvider; static PAGE_PROVIDER: RuntimePageProvider = RuntimePageProvider; static TASK_OPS: RuntimeTaskOps = RuntimeTaskOps; static BLOCK_IO_WAIT_WQ: ax_task::WaitQueue = ax_task::WaitQueue::new(); -static BLOCK_DRAIN_NOTIFY: ax_task::IrqNotify = ax_task::IrqNotify::new(); -#[cfg(feature = "irq")] +static BLOCK_DRAIN_NOTIFY_PENDING: AtomicBool = AtomicBool::new(false); +static BLOCK_DRAIN_WAIT_WQ: ax_task::WaitQueue = ax_task::WaitQueue::new(); +static BLOCK_DRAIN_IRQ_WAKER: spin::Once = spin::Once::new(); static IRQ_REGISTRAR: RuntimeBlockIrqRegistrar = RuntimeBlockIrqRegistrar; struct RuntimeTaskOps; @@ -76,15 +78,20 @@ impl BlockTaskOps for RuntimeTaskOps { } fn notify_drain(&self) { - BLOCK_DRAIN_NOTIFY.notify(); + BLOCK_DRAIN_NOTIFY_PENDING.store(true, Ordering::Release); + BLOCK_DRAIN_WAIT_WQ.notify_one(true); } fn notify_drain_from_irq(&self) { - BLOCK_DRAIN_NOTIFY.notify_irq(); + BLOCK_DRAIN_NOTIFY_PENDING.store(true, Ordering::Release); + if let Some(waker) = BLOCK_DRAIN_IRQ_WAKER.get() { + let _ = waker.wake_from_irq(0); + } } fn wait_for_drain_notification(&self) { - BLOCK_DRAIN_NOTIFY.wait(); + BLOCK_DRAIN_IRQ_WAKER.call_once(ax_task::current_hard_irq_waker); + BLOCK_DRAIN_WAIT_WQ.wait_until(|| BLOCK_DRAIN_NOTIFY_PENDING.swap(false, Ordering::AcqRel)); } fn spawn(&self, name: String, f: Box) { @@ -92,10 +99,8 @@ impl BlockTaskOps for RuntimeTaskOps { } } -#[cfg(feature = "irq")] struct RuntimeBlockIrqRegistrar; -#[cfg(feature = "irq")] struct RuntimeBlockIrqRegistration { _inner: crate::irq::Registration, } @@ -105,7 +110,6 @@ struct RuntimeBlockIrqRegistration { // the IRQ framework and is not exposed through this token. unsafe impl Sync for RuntimeBlockIrqRegistration {} -#[cfg(feature = "irq")] impl BlockIrqRegistration for RuntimeBlockIrqRegistration {} fn map_block_irq_error(err: ax_hal::irq::IrqError) -> ax_errno::AxError { @@ -126,7 +130,6 @@ fn map_block_irq_error(err: ax_hal::irq::IrqError) -> ax_errno::AxError { } } -#[cfg(feature = "irq")] impl BlockIrqRegistrar for RuntimeBlockIrqRegistrar { fn register_shared( &self, @@ -154,16 +157,10 @@ pub(super) fn init(bootargs: Option<&str>) { ax_fs_ng::root::init_root_from_rdif(take_rdif_block_devices(), bootargs); } -#[cfg(feature = "irq")] fn irq_registrar() -> Option<&'static dyn BlockIrqRegistrar> { Some(&IRQ_REGISTRAR) } -#[cfg(not(feature = "irq"))] -fn irq_registrar() -> Option<&'static dyn BlockIrqRegistrar> { - None -} - fn take_rdif_block_devices() -> Vec { ax_driver::block::take_rdif_block_devices() .into_iter() @@ -175,7 +172,6 @@ fn take_rdif_block_devices() -> Vec { .collect() } -#[cfg(feature = "irq")] fn resolve_block_irq(irq: Option) -> Option { let irq = irq?; match crate::irq::resolve_binding_irq(irq) { @@ -187,11 +183,6 @@ fn resolve_block_irq(irq: Option) -> Option) -> Option { - None -} - #[cfg(test)] mod tests { use super::*; diff --git a/os/arceos/modules/axruntime/src/klib.rs b/os/arceos/modules/axruntime/src/klib.rs index 0cb28ac204..e22170e548 100644 --- a/os/arceos/modules/axruntime/src/klib.rs +++ b/os/arceos/modules/axruntime/src/klib.rs @@ -3,8 +3,7 @@ //! This crate provides the platform-side glue that implements the small set //! of kernel helper functions defined in `axklib`. The implementation is //! intentionally minimal: it forwards memory mapping requests to `axmm`, -//! delegates timing to `ax-hal`, and wires IRQ operations to `ax-hal` when the -//! `irq` feature is enabled. +//! delegates timing to `ax-hal`, and wires IRQ operations to `ax-hal`. //! //! The implementation uses the `impl_trait!` helper to generate the FFI //! shims expected by consumers. Documentation here focuses on the behavior @@ -32,7 +31,6 @@ fn dma_coherent_range(addr: VirtAddr, size: usize) -> Option<(VirtAddr, usize)> Some((start, end - start)) } -#[cfg(feature = "irq")] fn map_irq_error(err: IrqError) -> AxError { match err { IrqError::InvalidIrq | IrqError::InvalidCpu => AxError::InvalidInput, @@ -203,119 +201,53 @@ impl_trait! { } /// Enable or disable the specified IRQ line. - /// - /// When the `irq` feature is enabled this forwards to - /// `ax_hal::irq::set_enable`. Platforms built without IRQ support - /// ignore this request because there is no interrupt controller - /// service to program. - fn irq_set_enable(_irq: IrqId, _enabled: bool) -> AxResult { - #[cfg(feature = "irq")] - { - ax_hal::irq::set_enable(_irq, _enabled).map_err(map_irq_error) - } - #[cfg(not(feature = "irq"))] - { - Err(AxError::Unsupported) - } + fn irq_set_enable(irq: IrqId, enabled: bool) -> AxResult { + ax_hal::irq::set_enable(irq, enabled).map_err(map_irq_error) } - fn irq_request_shared( - _irq: IrqId, - _handler: BoxedIrqHandler, - ) -> AxResult { - #[cfg(feature = "irq")] - { - ax_hal::irq::request_shared_irq(_irq, _handler).map_err(map_irq_error) - } - #[cfg(not(feature = "irq"))] - { - Err(AxError::Unsupported) - } + fn irq_request_shared(irq: IrqId, handler: BoxedIrqHandler) -> AxResult { + ax_hal::irq::request_shared_irq(irq, handler).map_err(map_irq_error) } fn irq_request_shared_disabled( - _irq: IrqId, - _handler: BoxedIrqHandler, + irq: IrqId, + handler: BoxedIrqHandler, ) -> AxResult { - #[cfg(feature = "irq")] - { - ax_hal::irq::request_irq( - _irq, - ax_hal::irq::IrqRequest::new(_handler) - .share_mode(ax_hal::irq::ShareMode::Shared) - .auto_enable(ax_hal::irq::AutoEnable::No), - ) - .map_err(map_irq_error) - } - #[cfg(not(feature = "irq"))] - { - Err(AxError::Unsupported) - } + ax_hal::irq::request_irq( + irq, + ax_hal::irq::IrqRequest::new(handler) + .share_mode(ax_hal::irq::ShareMode::Shared) + .auto_enable(ax_hal::irq::AutoEnable::No), + ) + .map_err(map_irq_error) } fn irq_request_percpu( - _irq: IrqId, - _cpus: IrqCpuMask, - _handler: ConcurrentBoxedIrqHandler, + irq: IrqId, + cpus: IrqCpuMask, + handler: ConcurrentBoxedIrqHandler, ) -> AxResult { - #[cfg(feature = "irq")] - { - ax_hal::irq::request_percpu_irq(_irq, _cpus, _handler) - .map_err(map_irq_error) - } - #[cfg(not(feature = "irq"))] - { - Err(AxError::Unsupported) - } + ax_hal::irq::request_percpu_irq(irq, cpus, handler).map_err(map_irq_error) } - fn irq_free(_handle: IrqHandle) -> AxResult { - #[cfg(feature = "irq")] - { - ax_hal::irq::free_irq(_handle).map_err(map_irq_error) - } - #[cfg(not(feature = "irq"))] - { - Err(AxError::Unsupported) - } + fn irq_free(handle: IrqHandle) -> AxResult { + ax_hal::irq::free_irq(handle).map_err(map_irq_error) } - fn irq_enable(_handle: IrqHandle) -> AxResult { - #[cfg(feature = "irq")] - { - ax_hal::irq::enable_irq(_handle).map_err(map_irq_error) - } - #[cfg(not(feature = "irq"))] - { - Err(AxError::Unsupported) - } + fn irq_enable(handle: IrqHandle) -> AxResult { + ax_hal::irq::enable_irq(handle).map_err(map_irq_error) } - fn irq_disable(_handle: IrqHandle) -> AxResult { - #[cfg(feature = "irq")] - { - ax_hal::irq::disable_irq(_handle).map_err(map_irq_error) - } - #[cfg(not(feature = "irq"))] - { - Err(AxError::Unsupported) - } + fn irq_disable(handle: IrqHandle) -> AxResult { + ax_hal::irq::disable_irq(handle).map_err(map_irq_error) } unsafe fn irq_run_on_cpu_sync( - _cpu: IrqCpuId, - _f: unsafe fn(*mut ()), - _arg: *mut (), + cpu: IrqCpuId, + f: unsafe fn(*mut ()), + arg: *mut (), ) -> Result<(), IrqError> { - #[cfg(feature = "irq")] - { - unsafe { ax_hal::irq::run_on_cpu_sync(_cpu, _f, _arg) } - } - #[cfg(not(feature = "irq"))] - { - let _ = (_cpu, _f, _arg); - Err(IrqError::Unsupported) - } + unsafe { ax_hal::irq::run_on_cpu_sync(cpu, f, arg) } } } } diff --git a/os/arceos/modules/axruntime/src/lib.rs b/os/arceos/modules/axruntime/src/lib.rs index 43092295f5..4cec51df1b 100644 --- a/os/arceos/modules/axruntime/src/lib.rs +++ b/os/arceos/modules/axruntime/src/lib.rs @@ -20,7 +20,6 @@ //! # Cargo Features //! //! - `paging`: Enable page table manipulation support. -//! - `irq`: Enable interrupt handling support. //! - `multitask`: Enable multi-threading support. //! - `smp`: Enable SMP (symmetric multiprocessing) support. //! - `fs`: Enable filesystem support. @@ -50,12 +49,10 @@ mod stack_protector; #[cfg(feature = "smp")] mod mp; -#[cfg(any(feature = "irq", feature = "paging"))] mod klib; mod devices; mod fs; -#[cfg(feature = "irq")] pub mod irq; mod registers; @@ -81,7 +78,6 @@ pub(crate) fn runtime_default_task_stack_size() -> usize { build_info::TASK_STACK_SIZE } -#[cfg(feature = "irq")] fn ticks_per_sec() -> u64 { build_info::TICKS_PER_SEC as u64 } @@ -277,11 +273,9 @@ pub fn rust_main(cpu_id: usize, arg: usize) -> ! { #[cfg(feature = "ipi")] { ax_ipi::init(); - #[cfg(feature = "irq")] ax_hal::irq::set_run_on_cpu_sync(ax_ipi_run_on_cpu_sync); } - #[cfg(feature = "irq")] { info!("Initialize interrupt handlers..."); init_interrupt(); @@ -335,7 +329,7 @@ pub fn rust_main(cpu_id: usize, arg: usize) -> ! { core::hint::spin_loop(); } - #[cfg(all(feature = "irq", feature = "ipi"))] + #[cfg(feature = "ipi")] ax_ipi::wait_for_all_cpus_ready(); ax_app_entry(); @@ -396,7 +390,6 @@ fn init_allocator() { } } -#[cfg(feature = "irq")] fn init_interrupt() { init_percpu_irq(ax_hal::percpu::this_cpu_id()); @@ -407,7 +400,6 @@ fn init_interrupt() { ax_ipi::mark_current_cpu_ready(); } -#[cfg(feature = "irq")] pub(crate) fn init_percpu_irq(cpu_id: usize) { ax_hal::irq::cpu_online(cpu_id).expect("failed to mark CPU online for IRQ framework"); ax_hal::irq::init_common_irq_handler(); @@ -425,7 +417,7 @@ pub(crate) fn init_percpu_irq(cpu_id: usize) { init_timer(); } -#[cfg(all(feature = "irq", feature = "ipi"))] +#[cfg(feature = "ipi")] unsafe fn ax_ipi_run_on_cpu_sync( cpu: usize, f: unsafe fn(*mut ()), @@ -434,16 +426,13 @@ unsafe fn ax_ipi_run_on_cpu_sync( unsafe { ax_ipi::run_on_cpu_sync_raw(cpu, f, arg) } } -#[cfg(feature = "irq")] fn periodic_interval_nanos() -> u64 { ax_hal::time::NANOS_PER_SEC / ticks_per_sec() } -#[cfg(feature = "irq")] #[ax_percpu::def_percpu] static NEXT_PERIODIC_DEADLINE_NANOS: u64 = 0; -#[cfg(feature = "irq")] fn init_timer() { ax_hal::time::enable_timer_irq(); let now_ns = ax_hal::time::monotonic_time_nanos(); @@ -454,7 +443,6 @@ fn init_timer() { program_next_timer(); } -#[cfg(feature = "irq")] fn advance_periodic_timer(now_ns: u64) -> bool { let mut deadline = unsafe { NEXT_PERIODIC_DEADLINE_NANOS.read_current_raw() }; if deadline == 0 { @@ -478,7 +466,6 @@ fn advance_periodic_timer(now_ns: u64) -> bool { true } -#[cfg(feature = "irq")] fn program_next_timer() { let mut deadline = unsafe { NEXT_PERIODIC_DEADLINE_NANOS.read_current_raw() }; if deadline == 0 { @@ -496,7 +483,6 @@ fn program_next_timer() { ax_task::note_programmed_timer_deadline_nanos(deadline); } -#[cfg(feature = "irq")] fn timer_irq_handler(ctx: ax_hal::irq::IrqContext) -> ax_hal::irq::IrqReturn { let _ = ctx; #[cfg(feature = "multitask")] @@ -509,13 +495,13 @@ fn timer_irq_handler(ctx: ax_hal::irq::IrqContext) -> ax_hal::irq::IrqReturn { ax_hal::irq::IrqReturn::Handled } -#[cfg(all(feature = "irq", feature = "ipi"))] +#[cfg(feature = "ipi")] fn ipi_irq_handler(_ctx: ax_hal::irq::IrqContext) -> ax_hal::irq::IrqReturn { ax_ipi::ipi_handler(); ax_hal::irq::IrqReturn::Handled } -#[cfg(all(feature = "irq", feature = "wake-ipi", not(feature = "ipi")))] +#[cfg(all(feature = "wake-ipi", not(feature = "ipi")))] fn ipi_irq_handler(_ctx: ax_hal::irq::IrqContext) -> ax_hal::irq::IrqReturn { ax_hal::irq::IrqReturn::Handled } diff --git a/os/arceos/modules/axruntime/src/mp.rs b/os/arceos/modules/axruntime/src/mp.rs index 2f40b5beeb..d214d2063d 100644 --- a/os/arceos/modules/axruntime/src/mp.rs +++ b/os/arceos/modules/axruntime/src/mp.rs @@ -95,13 +95,11 @@ pub fn rust_main_secondary(cpu_id: usize) -> ! { // Bring up local IRQ/IPI delivery before publishing INITED_CPUS so the // primary cannot enter user-visible init while remote CPUs still lack SGI // handlers or pending per-CPU IRQ enables. - #[cfg(feature = "irq")] super::init_percpu_irq(cpu_id); - #[cfg(feature = "irq")] ax_hal::asm::enable_irqs(); - #[cfg(all(feature = "irq", feature = "ipi"))] + #[cfg(feature = "ipi")] ax_ipi::mark_current_cpu_ready(); info!("Secondary CPU {cpu_id:x} init OK."); diff --git a/os/arceos/modules/axtask/Cargo.toml b/os/arceos/modules/axtask/Cargo.toml index 9c9050b789..52851e49a5 100644 --- a/os/arceos/modules/axtask/Cargo.toml +++ b/os/arceos/modules/axtask/Cargo.toml @@ -10,12 +10,16 @@ version = "0.5.24" [features] default = [] -irq = ["ax-hal/irq", "dep:ax-timer-list"] -ipi = ["irq", "ax-hal/ipi", "dep:ax-ipi"] +# Full remote callback IPI support. This includes the lightweight IRQ-wake IPI +# path plus the ax-ipi callback queue used by scheduler reschedule requests. +ipi = ["irq-wake-ipi", "dep:ax-ipi"] +# Lightweight hardware IPI support for hard-IRQ task wake. This wakes the target +# CPU so its IRQ epilogue can drain the per-CPU IRQ wake queue, but does not +# enable ax-ipi remote callback execution. +irq-wake-ipi = ["ax-hal/ipi"] multitask = [ "stack-canary", "dep:axpoll", - "dep:ax-sched", "dep:ax-cpumask", "dep:ax-crate-interface", "dep:futures-util", @@ -24,9 +28,11 @@ multitask = [ "dep:ax-lazyinit", "dep:ax-memory-addr", "dep:ax-percpu", + "dep:bare-task", "dep:spin", + "dep:ax-timer-list", ] -preempt = ["irq", "ax-percpu?/preempt", "ax-kernel-guard/preempt"] +preempt = ["ax-percpu?/preempt", "ax-kernel-guard/preempt"] smp = ["ax-kspin/smp"] stack-canary = [] stack-guard-page = ["multitask", "dep:ax-alloc", "dep:ax-mm", "ax-hal/paging"] @@ -72,9 +78,9 @@ ax-lazyinit = {workspace = true, optional = true} ax-memory-addr = {workspace = true, optional = true} ax-mm = {workspace = true, optional = true} ax-percpu = {workspace = true, optional = true} -ax-sched = { workspace = true, optional = true } ax-timer-list = {workspace = true, optional = true} axpoll = {workspace = true, optional = true} +bare-task = {workspace = true, optional = true} cfg-if.workspace = true extern-trait = {version = "0.4", optional = true} futures-util = {version = "0.3", default-features = false, optional = true, features = [ diff --git a/os/arceos/modules/axtask/src/api.rs b/os/arceos/modules/axtask/src/api.rs index d44f9960e8..e2cff8b280 100644 --- a/os/arceos/modules/axtask/src/api.rs +++ b/os/arceos/modules/axtask/src/api.rs @@ -18,8 +18,6 @@ pub(crate) use crate::run_queue::{current_run_queue, select_run_queue, select_wa #[cfg_attr(doc, doc(cfg(all(feature = "multitask", feature = "task-ext"))))] #[cfg(feature = "task-ext")] pub use crate::task::{AxTaskExt, TaskExt}; -#[cfg_attr(doc, doc(cfg(all(feature = "multitask", feature = "irq"))))] -#[cfg(feature = "irq")] pub use crate::timers::register_timer_callback; #[cfg_attr(doc, doc(cfg(feature = "multitask")))] pub use crate::{ @@ -48,15 +46,15 @@ pub fn default_task_stack_size() -> usize { cfg_if::cfg_if! { if #[cfg(feature = "sched-rr")] { const MAX_TIME_SLICE: usize = 5; - pub(crate) type AxTask = ax_sched::RRTask; - pub(crate) type Scheduler = ax_sched::RRScheduler; + pub(crate) type AxTask = bare_task::RRTask; + pub(crate) type Scheduler = bare_task::RRScheduler; } else if #[cfg(feature = "sched-cfs")] { - pub(crate) type AxTask = ax_sched::CFSTask; - pub(crate) type Scheduler = ax_sched::CFScheduler; + pub(crate) type AxTask = bare_task::CFSTask; + pub(crate) type Scheduler = bare_task::CFScheduler; } else { // If no scheduler features are set, use FIFO as the default. - pub(crate) type AxTask = ax_sched::FifoTask; - pub(crate) type Scheduler = ax_sched::FifoScheduler; + pub(crate) type AxTask = bare_task::FifoTask; + pub(crate) type Scheduler = bare_task::FifoScheduler; } } @@ -115,6 +113,15 @@ impl ax_kspin::lockdep::KspinLockdepIf for KspinLockdepIfImpl { } } +struct IrqEpilogueIfImpl; + +#[ax_crate_interface::impl_interface] +impl ax_hal::irq::IrqEpilogueIf for IrqEpilogueIfImpl { + fn drain_irq_wake_queue_current_cpu() -> usize { + crate::drain_irq_wake_queue_current_cpu() + } +} + /// Gets the current task, or returns [`None`] if the current task is not /// initialized. pub fn current_may_uninit() -> Option { @@ -174,18 +181,16 @@ pub fn init_scheduler_secondary(stack_ptr: VirtAddr, stack_size: usize) { /// Handles periodic timer ticks for the task manager. /// /// For example, advance scheduler states, checks timed events, etc. -#[cfg(feature = "irq")] -#[cfg_attr(doc, doc(cfg(feature = "irq")))] pub fn on_timer_tick() { on_timer_irq(true); } /// Handles a hardware timer interrupt. -#[cfg(feature = "irq")] -#[cfg_attr(doc, doc(cfg(feature = "irq")))] pub fn on_timer_irq(scheduler_tick: bool) { use ax_kernel_guard::NoOp; + current_run_queue::().mark_timer_irq_pending(); crate::timers::check_events(scheduler_tick); + current_run_queue::().clear_timer_service_pending(); if scheduler_tick { // Since irq and preemption are both disabled here, // we can get current run queue with the default `ax_kernel_guard::NoOp`. @@ -193,13 +198,11 @@ pub fn on_timer_irq(scheduler_tick: bool) { } } -#[cfg(feature = "irq")] #[doc(hidden)] pub fn next_timer_deadline_nanos() -> Option { crate::timers::next_deadline_nanos() } -#[cfg(feature = "irq")] #[doc(hidden)] pub fn note_programmed_timer_deadline_nanos(deadline_nanos: u64) { crate::timers::note_programmed_deadline_nanos(deadline_nanos); @@ -313,8 +316,6 @@ pub(crate) fn yield_now_unchecked() { } /// Current task is going to sleep for the given duration. -/// -/// If the feature `irq` is not enabled, it uses busy-wait instead. #[track_caller] pub fn sleep(dur: core::time::Duration) { sleep_until(ax_hal::time::monotonic_time() + dur); @@ -322,16 +323,10 @@ pub fn sleep(dur: core::time::Duration) { /// Current task is going to sleep, it will be woken up at the given deadline. /// The deadline is measured against the monotonic clock. -/// -/// If the feature `irq` is not enabled, it uses busy-wait instead. #[track_caller] pub fn sleep_until(deadline: ax_hal::time::TimeValue) { - #[cfg(feature = "irq")] might_sleep(); - #[cfg(feature = "irq")] current_run_queue::().sleep_until(deadline); - #[cfg(not(feature = "irq"))] - ax_hal::time::busy_wait_until(deadline); } /// Exits the current task. @@ -343,14 +338,7 @@ pub fn exit(exit_code: i32) -> ! { } fn current_irq_context() -> bool { - #[cfg(feature = "irq")] - { - ax_hal::irq::in_irq_context() - } - #[cfg(not(feature = "irq"))] - { - false - } + ax_hal::irq::in_irq_context() } #[derive(Clone, Copy)] @@ -430,19 +418,8 @@ impl AtomicContextSnapshot { } fn reasons(self) -> AtomicContextReasons { - let irq_disabled = { - #[cfg(feature = "irq")] - { - !self.irq_enabled - } - #[cfg(not(feature = "irq"))] - { - false - } - }; - AtomicContextReasons { - irq_disabled, + irq_disabled: !self.irq_enabled, irq_context: self.irq_context, preempt_disabled: self.preempt_count != 0, } @@ -531,18 +508,18 @@ pub fn wake_task(task: &AxTaskRef) { // unblocks the task via the registered waker callback. task.interrupt(); - // For tasks blocked on a raw WaitQueue, interrupt_waker.wake() is a - // no-op (no waker registered). Force-unblock by transitioning the task - // from Blocked to Ready and placing it on the run queue of its - // affinity CPU. + // For tasks blocked on a raw WaitQueue, interrupt_waker.wake() is a no-op + // (no waker registered). Clear the unknown wait-queue membership before + // force-unblocking so the original queue will discard its stale node later + // instead of consuming a future notify for this already-ready task. // // SAFETY: unblock_task uses a CAS on the task state (Blocked → Ready), // so if the task is concurrently being woken by its WaitQueue, the CAS - // fails and this is a harmless no-op. The stale entry in the WaitQueue - // is benign: when WaitQueue::notify_one eventually pops it, the - // subsequent unblock_task call will again CAS-fail (task already Ready - // or Running). + // fails and this is a harmless no-op. Clearing the wait-queue key is also + // harmless in that race: a waiter popped by its queue has already cleared + // the same key, and a remaining queue node is stale by construction. if task.state() == TaskState::Blocked { + task.clear_wait_queue_membership(); let mut rq = select_run_queue::(task); rq.unblock_task(task.clone(), false); } @@ -602,7 +579,7 @@ pub fn run_idle() -> ! { loop { yield_now_unchecked(); trace!("idle task: waiting for IRQs..."); - #[cfg(all(feature = "irq", not(feature = "host-test")))] + #[cfg(not(feature = "host-test"))] ax_hal::asm::wait_for_irqs(); } } diff --git a/os/arceos/modules/axtask/src/api_s.rs b/os/arceos/modules/axtask/src/api_s.rs index 61828fe705..3d04a74e5a 100644 --- a/os/arceos/modules/axtask/src/api_s.rs +++ b/os/arceos/modules/axtask/src/api_s.rs @@ -3,11 +3,7 @@ /// For single-task situation, we just relax the CPU and wait for incoming /// interrupts. pub fn yield_now() { - if cfg!(feature = "irq") { - ax_hal::asm::wait_for_irqs(); - } else { - core::hint::spin_loop(); - } + ax_hal::asm::wait_for_irqs(); } /// For single-task situation, we just busy wait for the given duration. diff --git a/os/arceos/modules/axtask/src/bare_adapter.rs b/os/arceos/modules/axtask/src/bare_adapter.rs new file mode 100644 index 0000000000..f3ac9b04d5 --- /dev/null +++ b/os/arceos/modules/axtask/src/bare_adapter.rs @@ -0,0 +1,88 @@ +//! Adapter from `bare-task` OS hooks to ArceOS task runtime services. + +use ax_kernel_guard::BaseGuard; +use bare_task::{BareTaskOs, CpuId, impl_trait}; + +struct BareTaskOsImpl; + +impl_trait! { + impl BareTaskOs for BareTaskOsImpl { + fn cpu_num() -> usize { + ax_hal::cpu_num() + } + + fn this_cpu_id() -> CpuId { + CpuId(ax_hal::percpu::this_cpu_id()) + } + + fn current_task_ptr() -> *const () { + ax_hal::percpu::current_task_ptr::<()>() + } + + unsafe fn set_current_task_ptr(ptr: *const ()) { + unsafe { ax_hal::percpu::set_current_task_ptr(ptr) }; + } + + fn irq_save_and_disable() -> usize { + ax_kernel_guard::IrqSave::acquire() + } + + unsafe fn irq_restore(state: usize) { + ax_kernel_guard::IrqSave::release(state); + } + + fn irqs_enabled() -> bool { + ax_hal::asm::irqs_enabled() + } + + fn in_irq_context() -> bool { + ax_hal::irq::in_irq_context() + } + + fn wait_for_irqs() { + ax_hal::asm::wait_for_irqs(); + } + + fn monotonic_time_nanos() -> u64 { + ax_hal::time::monotonic_time_nanos() + } + + fn set_oneshot_timer(deadline_nanos: u64) { + ax_hal::time::set_oneshot_timer(deadline_nanos); + } + + fn request_reschedule(cpu: CpuId) { + #[cfg(feature = "ipi")] + if cpu.0 != ax_hal::percpu::this_cpu_id() { + ax_ipi::run_on_cpu(cpu.0, || {}); + } + #[cfg(not(feature = "ipi"))] + let _ = cpu; + } + + fn request_irq_wake(cpu: CpuId) { + #[cfg(any(feature = "ipi", feature = "irq-wake-ipi"))] + if cpu.0 != ax_hal::percpu::this_cpu_id() { + ax_hal::irq::send_ipi( + ax_hal::irq::ipi_irq(), + ax_hal::irq::IpiTarget::Other { cpu_id: cpu.0 }, + ); + } + #[cfg(not(any(feature = "ipi", feature = "irq-wake-ipi")))] + let _ = cpu; + } + + fn wait_until_cpu_ready(cpu: CpuId) -> bool { + #[cfg(feature = "ipi")] + { + ax_ipi::wait_until_cpu_ready(cpu.0) + } + #[cfg(not(feature = "ipi"))] + { + cpu.0 == ax_hal::percpu::this_cpu_id() + } + } + + fn on_sched_switch(_prev: usize, _next: usize, _next_state: u8) {} + } +} diff --git a/os/arceos/modules/axtask/src/future/mod.rs b/os/arceos/modules/axtask/src/future/mod.rs index 6eb353479d..1e9a9c766c 100644 --- a/os/arceos/modules/axtask/src/future/mod.rs +++ b/os/arceos/modules/axtask/src/future/mod.rs @@ -1,18 +1,12 @@ //! Future support. -use alloc::{sync::Arc, task::Wake}; -use core::{ - fmt, - future::poll_fn, - pin::pin, - task::{Context, Poll, Waker}, -}; +use core::{fmt, future::poll_fn, pin::pin, task::Poll}; use ax_errno::AxError; use ax_kernel_guard::NoPreemptIrqSave; -use ax_kspin::SpinNoIrq; +use bare_task::{BlockOnTaskWake, BlockOnThreadWaker}; -use crate::{AxTaskRef, WeakAxTaskRef, current, current_run_queue, select_wake_run_queue}; +use crate::{TaskWaker, current, current_run_queue}; mod poll; pub use poll::*; @@ -20,31 +14,16 @@ pub use poll::*; mod time; pub use time::*; -struct AxWaker { - task: WeakAxTaskRef, - woke: SpinNoIrq, -} - -impl AxWaker { - fn new(task: &AxTaskRef) -> Arc { - Arc::new(AxWaker { - task: Arc::downgrade(task), - woke: SpinNoIrq::new(false), - }) - } -} +#[derive(Clone)] +struct AxBlockOnWake(TaskWaker); -impl Wake for AxWaker { - fn wake(self: Arc) { - self.wake_by_ref(); +impl BlockOnTaskWake for AxBlockOnWake { + fn wake_task(&self) { + let _ = self.0.wake(0); } - fn wake_by_ref(self: &Arc) { - if let Some(task) = self.task.upgrade() { - let mut rq = select_wake_run_queue::(&task); - *self.woke.lock() = true; - rq.unblock_task(task, true); - } + fn wake_seq(&self) -> u64 { + self.0.seq() } } @@ -64,11 +43,12 @@ pub fn block_on(f: F) -> F::Output { let curr = current(); let task = curr.clone(); - let axwaker = AxWaker::new(&task); - let waker = Waker::from(axwaker.clone()); - let mut cx = Context::from_waker(&waker); + let axwaker = BlockOnThreadWaker::new(AxBlockOnWake(TaskWaker::new(task.clone()))); + let waker = axwaker.waker(); + let mut cx = core::task::Context::from_waker(&waker); loop { + let observed_irq_seq = axwaker.wake_seq(); match fut.as_mut().poll(&mut cx) { Poll::Pending => { // Before sleeping, check if a signal has arrived. If so, @@ -82,15 +62,17 @@ pub fn block_on(f: F) -> F::Output { continue; } + if axwaker.should_repoll(observed_irq_seq) { + crate::yield_now(); + continue; + } + let mut rq = current_run_queue::(); - let mut woke = axwaker.woke.lock(); - if !*woke { - rq.future_blocked_resched(woke); - } else { - *woke = false; - drop(woke); + if axwaker.should_repoll(observed_irq_seq) { drop(rq); crate::yield_now(); + } else { + rq.future_blocked_resched(|| axwaker.should_repoll(observed_irq_seq)); } } Poll::Ready(output) => break output, diff --git a/os/arceos/modules/axtask/src/future/poll.rs b/os/arceos/modules/axtask/src/future/poll.rs index 0f73585936..50383ce89d 100644 --- a/os/arceos/modules/axtask/src/future/poll.rs +++ b/os/arceos/modules/axtask/src/future/poll.rs @@ -1,9 +1,54 @@ -use core::{future::poll_fn, task::Poll}; +use alloc::{collections::BTreeMap, string::String, sync::Arc, vec::Vec}; +use core::{ + future::poll_fn, + sync::atomic::{AtomicBool, Ordering}, + task::Poll, +}; use ax_errno::{AxError, AxResult}; -use axpoll::{IoEvents, Pollable}; +use ax_kspin::SpinNoIrq; +use axpoll::{IoEvents, PollSet, Pollable}; -use crate::current; +use crate::{HardIrqSignal, current}; + +static IRQ_NOTIFY: HardIrqSignal = HardIrqSignal::new(); +static DRAIN_SPAWNED: AtomicBool = AtomicBool::new(false); +static IRQ_STATE: SpinNoIrq>> = + SpinNoIrq::new(BTreeMap::new()); + +struct IrqPollState { + pending: AtomicBool, + installed: AtomicBool, + poll: Arc, +} + +impl IrqPollState { + fn new() -> Self { + Self { + pending: AtomicBool::new(false), + installed: AtomicBool::new(false), + poll: Arc::new(PollSet::new()), + } + } + + fn handle_irq(&self) -> ax_hal::irq::IrqReturn { + self.pending.store(true, Ordering::Release); + IRQ_NOTIFY.notify_irq(); + ax_hal::irq::IrqReturn::Handled + } + + fn take_pending(&self) -> bool { + self.pending.swap(false, Ordering::AcqRel) + } + + fn mark_installing(&self) -> bool { + !self.installed.swap(true, Ordering::AcqRel) + } + + fn clear_installing(&self) { + self.installed.store(false, Ordering::Release); + } +} /// A helper to wrap a synchronous non-blocking I/O function into an /// asynchronous function. @@ -60,47 +105,11 @@ pub async fn poll_io AxResult, T>( /// there is an existing waiter), which can deadlock against the task /// the IRQ preempted and triggers the slab from interrupt context. /// -/// The IRQ hook here does only what is safe in interrupt context: -/// flip a per-IRQ pending bit and `notify_one` a [`WaitQueue`]. -/// `WaitQueue::notify_one` just pops from a `VecDeque` under a -/// `SpinNoIrq` (no allocation, deadlock-free because IRQs are -/// already disabled in the holding paths) and re-queues the drain -/// task. The drain task runs in normal task context and is the only -/// place that ever calls `PollSet::wake`. -#[cfg(feature = "irq")] +/// The IRQ hook here does only what is safe in interrupt context: flip an +/// already-allocated per-IRQ pending bit and poke a [`HardIrqSignal`]. The drain +/// task runs in normal task context and is the only place that locks the +/// registry or calls `PollSet::wake`. pub fn register_irq_waker(irq: ax_hal::irq::IrqId, waker: &core::task::Waker) -> AxResult<()> { - use alloc::{collections::BTreeMap, sync::Arc}; - use core::sync::atomic::{AtomicBool, Ordering}; - - use ax_kspin::SpinNoIrq; - use axpoll::PollSet; - - use crate::IrqNotify; - - static IRQ_NOTIFY: IrqNotify = IrqNotify::new(); - static DRAIN_SPAWNED: AtomicBool = AtomicBool::new(false); - static IRQ_STATE: SpinNoIrq> = - SpinNoIrq::new(BTreeMap::new()); - - struct IrqPollState { - pending: bool, - installed: bool, - poll: Arc, - } - - fn irq_waker_handler(ctx: ax_hal::irq::IrqContext) -> ax_hal::irq::IrqReturn { - // Runs in IRQ context with interrupts off. Only mark an already - // registered slot and notify the drain task. The map entry is created - // during task-context registration, so this path does not allocate. - if let Some(state) = IRQ_STATE.lock().get_mut(&ctx.irq) { - state.pending = true; - IRQ_NOTIFY.notify_irq(); - ax_hal::irq::IrqReturn::Handled - } else { - ax_hal::irq::IrqReturn::Unhandled - } - } - fn ensure_drain_spawned() { if DRAIN_SPAWNED .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -117,12 +126,11 @@ pub fn register_irq_waker(irq: ax_hal::irq::IrqId, waker: &core::task::Waker) -> // map lock, then drop the lock before invoking // `wake` (which can allocate and re-enter the // scheduler). - let mut to_wake: alloc::vec::Vec> = alloc::vec::Vec::new(); + let mut to_wake: Vec> = Vec::new(); { - let mut map = IRQ_STATE.lock(); - for state in map.values_mut() { - if state.pending { - state.pending = false; + let map = IRQ_STATE.lock(); + for state in map.values() { + if state.take_pending() { to_wake.push(state.poll.clone()); } } @@ -132,40 +140,48 @@ pub fn register_irq_waker(irq: ax_hal::irq::IrqId, waker: &core::task::Waker) -> } } }, - alloc::string::String::from("irq_waker_drain"), + String::from("irq_waker_drain"), 0x4000, ); } ensure_drain_spawned(); - let (poll, should_install) = { + let state = { let mut map = IRQ_STATE.lock(); - let state = map.entry(irq).or_insert_with(|| IrqPollState { - pending: false, - installed: false, - poll: Arc::new(PollSet::new()), - }); - if state.installed { - (state.poll.clone(), false) - } else { - state.installed = true; - (state.poll.clone(), true) - } + map.entry(irq) + .or_insert_with(|| Arc::new(IrqPollState::new())) + .clone() }; - unsafe { poll.register(waker, axpoll::IoEvents::all()) }; + unsafe { state.poll.register(waker, axpoll::IoEvents::all()) }; - if should_install { - ax_hal::irq::request_shared_irq(irq, irq_waker_handler) - .map_err(|_| AxError::Unsupported)?; + if state.mark_installing() { + let handler_state = state.clone(); + if ax_hal::irq::request_shared_irq(irq, move |_| handler_state.handle_irq()).is_err() { + state.clear_installing(); + return Err(AxError::Unsupported); + } } ax_hal::irq::set_enable(irq, true).map_err(|_| AxError::Unsupported) } /// Registers a waker for a temporary legacy numeric IRQ. -#[cfg(feature = "irq")] pub fn register_legacy_irq_waker(irq: usize, waker: &core::task::Waker) -> AxResult<()> { let irq = ax_hal::irq::try_legacy_irq(irq).map_err(|_| AxError::InvalidInput)?; register_irq_waker(irq, waker) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn irq_poll_handler_does_not_need_registry_lock() { + let state = Arc::new(IrqPollState::new()); + let _registry_guard = IRQ_STATE.lock(); + + assert_eq!(state.handle_irq(), ax_hal::irq::IrqReturn::Handled); + assert!(state.take_pending()); + } +} diff --git a/os/arceos/modules/axtask/src/future/time.rs b/os/arceos/modules/axtask/src/future/time.rs index c64fe3b05c..01772a5d4f 100644 --- a/os/arceos/modules/axtask/src/future/time.rs +++ b/os/arceos/modules/axtask/src/future/time.rs @@ -1,105 +1,100 @@ -use alloc::collections::BTreeMap; +use alloc::string::String; use core::{ fmt, pin::Pin, - task::{Context, Poll, Waker}, + sync::atomic::{AtomicBool, AtomicU64, Ordering}, + task::{Context, Poll}, time::Duration, }; use ax_errno::AxError; use ax_hal::time::{TimeValue, monotonic_time, wall_time}; +use ax_kspin::SpinNoIrq; +use bare_task::{TimerKey, TimerRuntimeCore}; use futures_util::{FutureExt, select_biased}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -struct TimerKey { - deadline: TimeValue, - key: u64, -} +static TIMER_RUNTIME: SpinNoIrq = SpinNoIrq::new(TimerRuntimeCore::new()); +static TIMER_SIGNAL: crate::HardIrqSignal = crate::HardIrqSignal::new(); +static TIMER_SERVICE_SPAWNED: AtomicBool = AtomicBool::new(false); +static TIMER_SERVICE_PENDING: AtomicBool = AtomicBool::new(false); +static NEXT_DEADLINE_NANOS: AtomicU64 = AtomicU64::new(0); -struct TimerRuntime { - key: u64, - wheel: BTreeMap, +#[allow(dead_code)] +pub(crate) fn check_timer_events() { + check_timer_events_at(deadline_to_nanos(monotonic_time())); } -impl TimerRuntime { - const fn new() -> Self { - TimerRuntime { - key: 0, - wheel: BTreeMap::new(), - } +fn check_timer_events_at(now_nanos: u64) { + let deadline = NEXT_DEADLINE_NANOS.load(Ordering::Acquire); + if deadline != 0 && deadline <= now_nanos && !TIMER_SERVICE_PENDING.swap(true, Ordering::AcqRel) + { + TIMER_SIGNAL.notify_irq(); } +} - fn add(&mut self, deadline: TimeValue) -> Option { - if deadline <= monotonic_time() { - return None; - } - - let key = TimerKey { - deadline, - key: self.key, - }; - self.wheel.insert(key, Waker::noop().clone()); - self.key += 1; - - Some(key) +pub(crate) fn next_timer_deadline() -> Option { + if TIMER_SERVICE_PENDING.load(Ordering::Acquire) { + return None; } + let deadline = NEXT_DEADLINE_NANOS.load(Ordering::Acquire); + (deadline != 0).then(|| TimeValue::from_nanos(deadline)) +} - fn poll(&mut self, key: &TimerKey, cx: &mut Context<'_>) -> Poll<()> { - if let Some(w) = self.wheel.get_mut(key) { - *w = cx.waker().clone(); - Poll::Pending - } else { - Poll::Ready(()) - } - } +fn with_runtime(f: impl FnOnce(&mut TimerRuntimeCore) -> R) -> R { + f(&mut TIMER_RUNTIME.lock()) +} - fn cancel(&mut self, key: &TimerKey) { - self.wheel.remove(key); - } +fn deadline_to_nanos(deadline: TimeValue) -> u64 { + deadline.as_nanos().min(u64::MAX as u128) as u64 +} - #[cfg(feature = "irq")] - fn next_deadline(&self) -> Option { - self.wheel.keys().next().map(|key| key.deadline) +fn update_next_deadline(runtime: &TimerRuntimeCore) { + let deadline = runtime.next_deadline_nanos().unwrap_or(0); + NEXT_DEADLINE_NANOS.store(deadline, Ordering::Release); +} + +fn ensure_timer_service_spawned() { + if TIMER_SERVICE_SPAWNED + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return; } - fn wake(&mut self) { - if self.wheel.is_empty() { - return; - } + crate::spawn_raw( + || loop { + TIMER_SIGNAL.wait(); + drain_expired_timers(); + }, + String::from("future-timer"), + crate::default_task_stack_size(), + ); +} +fn drain_expired_timers() { + loop { + TIMER_SERVICE_PENDING.store(false, Ordering::Release); let now = monotonic_time(); - - let pending = self.wheel.split_off(&TimerKey { - deadline: now, - key: u64::MAX, + let now_nanos = deadline_to_nanos(now); + let expired = with_runtime(|runtime| { + let expired = runtime.take_expired(now_nanos); + update_next_deadline(runtime); + expired }); - - let expired = core::mem::replace(&mut self.wheel, pending); - for (_, w) in expired { - w.wake(); + if expired.is_empty() { + break; + } + for waker in expired { + waker.wake(); + } + } + if let Some(deadline) = next_timer_deadline() { + if deadline <= monotonic_time() { + TIMER_SIGNAL.notify(); + } else { + crate::timers::maybe_reprogram_timer(deadline); } } -} - -percpu_static! { - TIMER_RUNTIME: TimerRuntime = TimerRuntime::new(), -} - -#[allow(dead_code)] -pub(crate) fn check_timer_events() { - // SAFETY: only called in timer::check_events - unsafe { TIMER_RUNTIME.current_ref_mut_raw() }.wake(); -} - -#[cfg(feature = "irq")] -pub(crate) fn next_timer_deadline() -> Option { - with_current(|r| r.next_deadline()) -} - -fn with_current(f: impl FnOnce(&mut TimerRuntime) -> R) -> R { - // FIXME: optimize `ax-percpu` crate! should disable irq and provide more apis - let _g = ax_kernel_guard::NoPreemptIrqSave::new(); - f(unsafe { TIMER_RUNTIME.current_ref_mut_raw() }) } /// Future returned by `sleep` and `sleep_until`. @@ -110,13 +105,16 @@ impl Future for TimerFuture { type Output = (); fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - with_current(|r| r.poll(&self.0, cx)) + with_runtime(|r| r.poll(&self.0, cx.waker())) } } impl Drop for TimerFuture { fn drop(&mut self) { - with_current(|r| r.cancel(&self.0)); + with_runtime(|runtime| { + runtime.cancel(&self.0); + update_next_deadline(runtime); + }); } } @@ -127,9 +125,15 @@ pub async fn sleep(duration: Duration) { /// Waits until the monotonic `deadline` is reached. pub async fn sleep_until(deadline: TimeValue) { - let key = with_current(|r| r.add(deadline)); + ensure_timer_service_spawned(); + let deadline_nanos = deadline_to_nanos(deadline); + let now_nanos = deadline_to_nanos(monotonic_time()); + let key = with_runtime(|runtime| { + let key = runtime.add(deadline_nanos, now_nanos); + update_next_deadline(runtime); + key + }); if let Some(key) = key { - #[cfg(feature = "irq")] crate::timers::maybe_reprogram_timer(deadline); TimerFuture(key).await; } @@ -199,3 +203,63 @@ fn wall_deadline_to_monotonic(deadline: TimeValue) -> TimeValue { .unwrap_or(TimeValue::MAX) } } + +#[cfg(test)] +mod tests { + use alloc::{sync::Arc, task::Wake}; + use core::{ + sync::atomic::{AtomicUsize, Ordering}, + task::Waker, + }; + + use super::*; + + struct WakeCounter(AtomicUsize); + + impl Wake for WakeCounter { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::AcqRel); + } + + fn wake_by_ref(self: &Arc) { + self.0.fetch_add(1, Ordering::AcqRel); + } + } + + #[test] + fn timer_runtime_takes_expired_wakers_without_waking_under_lock() { + let mut runtime = TimerRuntimeCore::new(); + let counter = Arc::new(WakeCounter(AtomicUsize::new(0))); + let waker = Waker::from(counter.clone()); + + let deadline = monotonic_time() + Duration::from_millis(1); + let deadline_nanos = deadline_to_nanos(deadline); + let key = runtime + .add(deadline_nanos, deadline_to_nanos(monotonic_time())) + .expect("future timer should be armed"); + assert!(runtime.poll(&key, &waker).is_pending()); + + let expired = runtime.take_expired(deadline_to_nanos(deadline + Duration::from_millis(1))); + assert_eq!(counter.0.load(Ordering::Acquire), 0); + assert_eq!(expired.len(), 1); + + for waker in expired { + waker.wake(); + } + assert_eq!(counter.0.load(Ordering::Acquire), 1); + } + + #[test] + fn check_timer_events_marks_service_pending_without_losing_deadline() { + let expired = 1; + NEXT_DEADLINE_NANOS.store(expired, Ordering::Release); + TIMER_SERVICE_PENDING.store(false, Ordering::Release); + + check_timer_events_at(expired); + + assert_eq!(NEXT_DEADLINE_NANOS.load(Ordering::Acquire), expired); + assert!(TIMER_SERVICE_PENDING.load(Ordering::Acquire)); + let _ = TIMER_SIGNAL.drain(); + TIMER_SERVICE_PENDING.store(false, Ordering::Release); + } +} diff --git a/os/arceos/modules/axtask/src/irq_notify.rs b/os/arceos/modules/axtask/src/irq_notify.rs index feb9afcf71..32e3a906e8 100644 --- a/os/arceos/modules/axtask/src/irq_notify.rs +++ b/os/arceos/modules/axtask/src/irq_notify.rs @@ -1,31 +1,44 @@ -use core::sync::atomic::{AtomicBool, Ordering}; +use alloc::{sync::Arc, vec::Vec}; +use core::{ + ptr, + sync::atomic::{AtomicBool, AtomicPtr, Ordering}, +}; -use crate::WaitQueue; +use ax_kernel_guard::NoPreemptIrqSave; + +struct HardIrqSignalWaiter { + task_id: u64, + generation: u64, + irq_waker: crate::HardIrqWaker, + task_waker: crate::TaskWaker, +} /// IRQ-safe deferred notification primitive. /// -/// `IrqNotify` separates a hard-IRQ notification from the slow work that must +/// `HardIrqSignal` separates a hard-IRQ notification from the slow work that must /// run in task context. IRQ handlers call [`notify_irq`](Self::notify_irq) to /// publish a pending bit and wake a deferred worker. The worker then drains the /// bit and performs the expensive work, such as polling a device or waking /// `axpoll` waiters. -pub struct IrqNotify { +pub struct HardIrqSignal { pending: AtomicBool, - wait: WaitQueue, + active_waiter: AtomicPtr, + waiters: ax_kspin::SpinNoIrq>>, } -impl Default for IrqNotify { +impl Default for HardIrqSignal { fn default() -> Self { Self::new() } } -impl IrqNotify { +impl HardIrqSignal { /// Creates an empty notification object. pub const fn new() -> Self { Self { pending: AtomicBool::new(false), - wait: WaitQueue::new(), + active_waiter: AtomicPtr::new(ptr::null_mut()), + waiters: ax_kspin::SpinNoIrq::new(Vec::new()), } } @@ -36,7 +49,17 @@ impl IrqNotify { /// coalesce into one pending bit until a worker drains it. pub fn notify_irq(&self) { self.pending.store(true, Ordering::Release); - self.wait.notify_one_from_irq(); + self.wake_active_from_irq(); + } + + fn wake_active_from_irq(&self) { + let waiter = self.active_waiter.load(Ordering::Acquire); + if !waiter.is_null() { + // SAFETY: waiter nodes are ref-counted and retained in `self.waiters` + // until the `HardIrqSignal` is dropped. IRQ producers must still follow + // the owning device's normal disable/synchronize-before-drop rule. + let _ = unsafe { &*waiter }.irq_waker.wake_from_irq(0); + } } /// Publishes a pending notification from task context. @@ -46,7 +69,15 @@ impl IrqNotify { /// behavior while still allowing the scheduler to observe a normal wake. pub fn notify(&self) { self.pending.store(true, Ordering::Release); - self.wait.notify_one(true); + self.wake_active(); + } + + fn wake_active(&self) { + let waiter = self.active_waiter.load(Ordering::Acquire); + if !waiter.is_null() { + // SAFETY: see `wake_active_from_irq`. + let _ = unsafe { &*waiter }.task_waker.wake(0); + } } /// Returns whether a notification is currently pending. @@ -54,6 +85,15 @@ impl IrqNotify { self.pending.load(Ordering::Acquire) } + /// Arms the current task as the IRQ wake target without blocking. + /// + /// Use this when the actual sleep happens through another primitive, such as + /// a timeout wait queue whose predicate checks [`is_pending`](Self::is_pending). + #[track_caller] + pub fn arm_current_task(&self) { + self.init_hard_irq_waker(); + } + /// Drains the pending bit. /// /// Returns `true` if at least one notification was pending. @@ -64,6 +104,57 @@ impl IrqNotify { /// Blocks until at least one pending notification is available, then drains it. #[track_caller] pub fn wait(&self) { - self.wait.wait_until(|| self.drain()); + self.wait_until(|| false); + } + + /// Blocks until `condition` becomes true or at least one pending notification + /// is available, then drains the notification bit. + #[track_caller] + pub fn wait_until(&self, condition: impl Fn() -> bool) { + crate::api::might_sleep(); + self.init_hard_irq_waker(); + loop { + if self.should_stop_waiting(&condition) { + return; + } + crate::current_run_queue::() + .future_blocked_resched(|| self.should_stop_waiting(&condition)); + } + } + + fn should_stop_waiting(&self, condition: &impl Fn() -> bool) -> bool { + let ready = condition(); + let notified = self.drain(); + ready || notified + } + + fn init_hard_irq_waker(&self) { + self.arm_task_waker(crate::current_task_waker()); + } + + fn arm_task_waker(&self, waker: crate::TaskWaker) { + let task_id = waker.task_id(); + let generation = waker.generation(); + let mut waiters = self.waiters.lock(); + let waiter = if let Some(index) = waiters + .iter() + .position(|waiter| waiter.task_id == task_id && waiter.generation == generation) + { + Arc::as_ptr(&waiters[index]) as *mut HardIrqSignalWaiter + } else { + waiters.push(Arc::new(HardIrqSignalWaiter { + task_id, + generation, + irq_waker: waker.to_hard_irq_waker(), + task_waker: waker, + })); + Arc::as_ptr(waiters.last().expect("just pushed waiter")) as *mut HardIrqSignalWaiter + }; + self.active_waiter.store(waiter, Ordering::Release); + } + + #[cfg(test)] + pub(crate) fn arm_irq_waker_for_test(&self, waker: crate::HardIrqWaker) { + self.arm_task_waker(crate::TaskWaker::from_hard_irq_waker_for_test(waker)); } } diff --git a/os/arceos/modules/axtask/src/irq_wake.rs b/os/arceos/modules/axtask/src/irq_wake.rs new file mode 100644 index 0000000000..6fab03db15 --- /dev/null +++ b/os/arceos/modules/axtask/src/irq_wake.rs @@ -0,0 +1,343 @@ +//! Task wake support for both hard IRQ and task context. +//! +//! The hard-IRQ path only records state and links a preallocated per-task node +//! into a per-CPU pending list. Scheduler locks are acquired later by the drain +//! path running with the usual task/scheduler guards. Task-context wake uses a +//! separate handle that may call into the scheduler directly. + +use alloc::sync::Arc; +use core::sync::atomic::{AtomicBool, Ordering}; + +use ax_hal::percpu::this_cpu_id; +use ax_kernel_guard::NoPreemptIrqSave; +use ax_lazyinit::LazyInit; + +#[cfg(all(feature = "smp", any(feature = "ipi", feature = "irq-wake-ipi")))] +use crate::run_queue::kick_remote_cpu_for_irq_wake; +use crate::{AxTask, AxTaskRef, TaskInner, WeakAxTaskRef, current, current_may_uninit}; + +#[ax_percpu::def_percpu] +static IRQ_WAKE_QUEUE: LazyInit = LazyInit::new(); + +#[cfg(all(test, feature = "host-test"))] +static HOST_TEST_IRQ_WAKE_QUEUE: spin::Once = spin::Once::new(); + +#[ax_percpu::def_percpu] +static IRQ_WAKE_DRAINING: AtomicBool = AtomicBool::new(false); + +pub use bare_task::{WakeBits, WakeResult, WakeSeq}; + +#[derive(Clone)] +struct WakeHandle { + task: WeakAxTaskRef, + task_id: u64, + generation: u64, +} + +impl WakeHandle { + fn new(task: AxTaskRef) -> Self { + Self { + task_id: task.id().as_u64(), + generation: task.irq_wake_generation(), + task: Arc::downgrade(&task), + } + } + + fn valid_task(&self) -> Option { + let task = self.task.upgrade()?; + if task.id().as_u64() != self.task_id { + return None; + } + if !task.irq_wake_generation_matches(self.generation) { + return None; + } + Some(task) + } + + fn task_id(&self) -> u64 { + self.task_id + } + + const fn generation(&self) -> u64 { + self.generation + } + + fn seq(&self) -> WakeSeq { + self.task.upgrade().map_or(0, |task| task.irq_wake_seq()) + } + + fn take_bits(&self) -> WakeBits { + self.task + .upgrade() + .map_or(0, |task| task.take_irq_wake_bits()) + } +} + +/// Cloneable hard-IRQ-safe handle that wakes one kernel task. +/// +/// This type is safe to store inside boxed IRQ callbacks. It never calls +/// arbitrary Rust [`core::task::Waker`] implementations and never takes +/// scheduler or wait-queue locks from the IRQ callback. +#[derive(Clone)] +pub struct HardIrqWaker { + handle: WakeHandle, +} + +impl HardIrqWaker { + /// Returns the task id captured by this waker. + pub fn task_id(&self) -> u64 { + self.handle.task_id() + } + + /// Returns the task generation captured by this waker. + pub const fn generation(&self) -> u64 { + self.handle.generation() + } + + /// Returns the current wake sequence. + pub fn seq(&self) -> WakeSeq { + self.handle.seq() + } + + /// Takes coalesced wake bits. + pub fn take_bits(&self) -> WakeBits { + self.handle.take_bits() + } + + /// Wakes the captured task from hard IRQ context. + pub fn wake_from_irq(&self, bits: WakeBits) -> WakeResult { + let Some(task) = self.handle.valid_task() else { + return WakeResult::default(); + }; + task.publish_irq_wake_bits(bits); + task.bump_irq_wake_seq(); + if !task.mark_irq_wake_pending() { + return WakeResult::default(); + } + + let target_cpu = task.cpu_id() as usize; + let Some(queue) = irq_wake_queue_for_cpu(target_cpu) else { + task.take_irq_wake_pending(); + return WakeResult::default(); + }; + queue.push(&task); + #[cfg(all(test, feature = "host-test"))] + let local = true; + #[cfg(not(all(test, feature = "host-test")))] + let local = target_cpu == this_cpu_id(); + #[cfg(all(feature = "smp", any(feature = "ipi", feature = "irq-wake-ipi")))] + let remote = if !local { + kick_remote_cpu_for_irq_wake(target_cpu); + true + } else { + false + }; + #[cfg(not(all(feature = "smp", any(feature = "ipi", feature = "irq-wake-ipi"))))] + let remote = false; + WakeResult::new(true, local, remote) + } +} + +/// Cloneable task-context handle that wakes one kernel task. +/// +/// This type may take scheduler locks and must not be called from hard IRQ +/// callbacks. Use [`HardIrqWaker`] for callbacks registered with IRQ dispatch. +#[derive(Clone)] +pub struct TaskWaker { + handle: WakeHandle, +} + +impl TaskWaker { + pub(crate) fn new(task: AxTaskRef) -> Self { + Self { + handle: WakeHandle::new(task), + } + } + + #[cfg(test)] + pub(crate) fn from_hard_irq_waker_for_test(waker: HardIrqWaker) -> Self { + Self { + handle: waker.handle, + } + } + + /// Creates a hard-IRQ-safe handle for the same task and generation. + pub fn to_hard_irq_waker(&self) -> HardIrqWaker { + HardIrqWaker { + handle: self.handle.clone(), + } + } + + /// Returns the task id captured by this waker. + pub fn task_id(&self) -> u64 { + self.handle.task_id() + } + + /// Returns the task generation captured by this waker. + pub const fn generation(&self) -> u64 { + self.handle.generation() + } + + /// Returns the current wake sequence. + pub fn seq(&self) -> WakeSeq { + self.handle.seq() + } + + /// Takes coalesced wake bits. + pub fn take_bits(&self) -> WakeBits { + self.handle.take_bits() + } + + /// Wakes the captured task from task context. + pub fn wake(&self, bits: WakeBits) -> WakeResult { + let Some(task) = self.handle.valid_task() else { + return WakeResult::default(); + }; + task.publish_irq_wake_bits(bits); + task.bump_irq_wake_seq(); + + #[cfg(all(test, feature = "host-test"))] + let local = true; + #[cfg(not(all(test, feature = "host-test")))] + let local = task.cpu_id() as usize == this_cpu_id(); + + if current_may_uninit() + .as_ref() + .is_some_and(|current| current.ptr_eq(&task)) + && task.transition_state(crate::TaskState::Blocked, crate::TaskState::Running) + { + return WakeResult::new(true, local, false); + } + + let woke = crate::run_queue::wake_task_from_irq_queue(task); + WakeResult::new(woke, local, woke && !local) + } +} + +/// Returns a task-context waker for the current task. +pub fn current_task_waker() -> TaskWaker { + TaskWaker::new(current().clone()) +} + +/// Returns a task-context waker for the current task when task state is initialized. +pub fn try_current_task_waker() -> Option { + current_may_uninit().map(|task| TaskWaker::new(task.clone())) +} + +/// Returns a hard-IRQ-safe waker for the current task. +pub fn current_hard_irq_waker() -> HardIrqWaker { + current_task_waker().to_hard_irq_waker() +} + +/// Drains the current CPU's IRQ wake queue into the scheduler. +pub fn drain_irq_wake_queue_current_cpu() -> usize { + let _guard = NoPreemptIrqSave::new(); + #[cfg(all(feature = "smp", any(feature = "ipi", feature = "irq-wake-ipi")))] + crate::run_queue::clear_remote_irq_wake_pending_for_current_cpu(); + + let draining = unsafe { IRQ_WAKE_DRAINING.current_ref_raw() }; + if draining.swap(true, Ordering::AcqRel) { + return 0; + } + let Some(queue) = irq_wake_queue_for_cpu(this_cpu_id()) else { + draining.store(false, Ordering::Release); + return 0; + }; + let mut drained = 0; + loop { + while let Some(task) = queue.pop() { + task.clear_irq_wake_link(); + if !task.take_irq_wake_pending() { + continue; + } + if current_may_uninit() + .as_ref() + .is_some_and(|current| current.ptr_eq(&task)) + { + if task.transition_state(crate::TaskState::Blocked, crate::TaskState::Running) { + drained += 1; + } + continue; + } + if crate::run_queue::wake_task_from_irq_queue(task) { + drained += 1; + } + } + + draining.store(false, Ordering::Release); + if queue.is_empty() + || draining + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + break; + } + } + drained +} + +pub(crate) fn init_irq_wake_queue_current_cpu() { + IRQ_WAKE_QUEUE.with_current(|queue| { + if !queue.is_inited() { + queue.init_once(IrqWakeQueue::new()); + } + }); + #[cfg(all(test, feature = "host-test"))] + HOST_TEST_IRQ_WAKE_QUEUE.call_once(IrqWakeQueue::new); +} + +pub(crate) fn expire_task_irq_wakers(task: &TaskInner) { + task.expire_irq_wakers(); +} + +fn irq_wake_queue_for_cpu(cpu_id: usize) -> Option<&'static IrqWakeQueue> { + #[cfg(all(test, feature = "host-test"))] + { + let _ = cpu_id; + HOST_TEST_IRQ_WAKE_QUEUE.get() + } + #[cfg(all(feature = "smp", not(all(test, feature = "host-test"))))] + { + debug_assert!(cpu_id < crate::build_info::CPU_CAPACITY); + unsafe { IRQ_WAKE_QUEUE.remote_ref_raw(cpu_id) }.get() + } + #[cfg(all(not(feature = "smp"), not(all(test, feature = "host-test"))))] + { + let _ = cpu_id; + unsafe { IRQ_WAKE_QUEUE.current_ref_raw() }.get() + } +} + +struct IrqWakeQueue { + core: bare_task::IrqWakeQueueCore, +} + +impl IrqWakeQueue { + const fn new() -> Self { + Self { + core: bare_task::IrqWakeQueueCore::new(), + } + } + + fn push(&self, task: &AxTaskRef) { + let task_ptr = Arc::as_ptr(task) as *mut AxTask; + // Keep the task alive before publishing the raw pointer. Exactly one + // extra reference is consumed by the drain path with `Arc::from_raw`. + unsafe { Arc::increment_strong_count(task_ptr) }; + self.core.push(task_ptr.cast::<()>(), |next| { + task.set_irq_wake_next(next.cast::()); + }); + } + + fn pop(&self) -> Option { + let head = self.core.pop(|node| { + let task = unsafe { &*node.cast::() }; + task.irq_wake_next().cast::<()>() + })?; + Some(unsafe { Arc::from_raw(head.cast::()) }) + } + + fn is_empty(&self) -> bool { + self.core.is_empty() + } +} diff --git a/os/arceos/modules/axtask/src/lib.rs b/os/arceos/modules/axtask/src/lib.rs index 59dc2cb7a6..3c7ec6ce70 100644 --- a/os/arceos/modules/axtask/src/lib.rs +++ b/os/arceos/modules/axtask/src/lib.rs @@ -9,9 +9,6 @@ //! - `multitask`: Enable multi-task support. If it's enabled, complex task //! management and scheduling is used, as well as more task-related APIs. //! Otherwise, only a few APIs with naive implementation is available. -//! - `irq`: Interrupts are enabled. If this feature is enabled, timer-based -//! APIs can be used, such as [`sleep`], [`sleep_until`], and -//! [`WaitQueue::wait_timeout`]. //! - `preempt`: Enable preemptive scheduling. //! - `sched-fifo`: Use the [FIFO cooperative scheduler][1]. It also enables the //! `multitask` feature if it is enabled. This feature is enabled by default, @@ -22,9 +19,9 @@ //! the `multitask` and `preempt` features if it is enabled. //! - `host-test`: Use host-safe fallbacks for unit tests. //! -//! [1]: ax_sched::FifoScheduler -//! [2]: ax_sched::RRScheduler -//! [3]: ax_sched::CFScheduler +//! [1]: bare_task::FifoScheduler +//! [2]: bare_task::RRScheduler +//! [3]: bare_task::CFScheduler #![cfg_attr(any(not(test), target_os = "none"), no_std)] #![cfg_attr(all(test, target_os = "none"), no_main)] @@ -76,11 +73,12 @@ cfg_if::cfg_if! { mod lockdep; #[cfg(feature = "tracepoint-hooks")] mod sched_tracepoint; - #[cfg(feature = "irq")] mod irq_notify; + mod irq_wake; + mod bare_adapter; + pub mod local; mod wait_queue; - #[cfg(feature = "irq")] mod timers; #[cfg(feature = "multitask")] @@ -88,8 +86,17 @@ cfg_if::cfg_if! { #[cfg_attr(doc, doc(cfg(feature = "multitask")))] pub use self::api::*; - #[cfg(feature = "irq")] - pub use self::irq_notify::IrqNotify; + pub use self::irq_notify::HardIrqSignal; + pub mod wake { + pub use super::irq_wake::{ + HardIrqWaker, TaskWaker, WakeBits, WakeResult, WakeSeq, current_hard_irq_waker, + current_task_waker, try_current_task_waker, + }; + } + pub use self::irq_wake::{ + HardIrqWaker, TaskWaker, WakeBits, WakeResult, WakeSeq, current_hard_irq_waker, + current_task_waker, drain_irq_wake_queue_current_cpu, try_current_task_waker, + }; pub use self::api::{sleep, sleep_until, yield_now}; #[cfg(feature = "tracepoint-hooks")] pub use self::sched_tracepoint::SchedTracepoint; diff --git a/os/arceos/modules/axtask/src/local.rs b/os/arceos/modules/axtask/src/local.rs new file mode 100644 index 0000000000..e00b81f298 --- /dev/null +++ b/os/arceos/modules/axtask/src/local.rs @@ -0,0 +1,244 @@ +//! Local task runtime helpers. +//! +//! These primitives are intended for device/runtime worker threads that want to +//! run several cooperative futures inside one ordinary `ax-task` thread. The +//! outer scheduler remains the traditional thread scheduler; the local executor +//! only multiplexes futures while the host thread is running. + +use alloc::sync::Arc; +use core::{ + future::Future, + task::{Context, Poll}, +}; + +use bare_task::{LocalExecutorCore, LocalSpawnerCore}; +pub use bare_task::{RuntimeEventSeq, RuntimeEventValue, SpawnLocalError}; + +use crate::{HardIrqSignal, HardIrqWaker, WaitQueue}; + +/// Sticky event source for IRQ/deferred runtime code. +/// +/// `RuntimeEvent` stores readiness as a monotonically increasing sequence plus +/// coalesced event bits. Wakers are only a hint to re-poll; callers must still +/// consume the published state. +pub struct RuntimeEvent { + core: bare_task::RuntimeEventCore, + notify: HardIrqSignal, +} + +impl RuntimeEvent { + /// Creates an empty event source. + pub const fn new() -> Self { + Self { + core: bare_task::RuntimeEventCore::new(), + notify: HardIrqSignal::new(), + } + } + + /// Returns the latest sequence. + pub fn seq(&self) -> RuntimeEventValue { + self.core.seq() + } + + /// Returns whether at least one event has been published. + pub fn has_unseen_events(&self) -> bool { + self.core.has_unseen_events() + } + + /// Returns whether the event sequence has changed from `observed`. + pub fn has_changed(&self, observed: &RuntimeEventSeq) -> bool { + self.core.has_changed(observed) + } + + /// Blocks until the sequence differs from `observed`, then updates it. + #[track_caller] + pub fn wait_changed(&self, observed: &RuntimeEventSeq) -> RuntimeEventValue { + self.wait_until(|| self.has_changed(observed)); + let seq = self.seq(); + observed.update(seq); + seq + } + + /// Publishes event bits from task/deferred context. + pub fn publish(&self, bits: u64) -> RuntimeEventValue { + let seq = self.core.publish(bits); + self.notify.notify(); + seq + } + + /// Publishes event bits from hard IRQ context. + pub fn publish_from_irq(&self, bits: u64) -> RuntimeEventValue { + self.core.publish_state(bits) + } + + /// Publishes event bits from hard IRQ context and wakes a host task through + /// the IRQ-safe task wake path. + pub fn publish_from_irq_with( + &self, + bits: u64, + waker: &HardIrqWaker, + ) -> (RuntimeEventValue, crate::WakeResult) { + let seq = self.core.publish_state(bits); + let wake = waker.wake_from_irq(bits); + (seq, wake) + } + + /// Takes all coalesced event bits. + pub fn take_bits(&self) -> u64 { + self.core.take_bits() + } + + /// Polls until the event sequence changes from `observed`. + /// + /// The registration protocol is: check sequence, register, then re-check. + /// This mirrors the standard async lost-wake prevention pattern. + pub fn poll_changed(&self, observed: &RuntimeEventSeq, cx: &mut Context<'_>) -> Poll { + self.core.poll_changed(observed, cx) + } + + /// Blocks the current host thread until `condition` becomes true or a + /// runtime event is published. + #[track_caller] + pub fn wait_until(&self, condition: impl Fn() -> bool) { + self.notify.wait_until(condition); + } + + /// Wakes futures registered with [`poll_changed`](Self::poll_changed). + /// + /// IRQ publishers intentionally do not call arbitrary wakers. Device + /// runtime threads should call this after the IRQ notification has returned + /// them to task context. + pub fn wake_waiters_deferred(&self) { + self.core.wake_waiters(); + } +} + +impl Default for RuntimeEvent { + fn default() -> Self { + Self::new() + } +} + +/// Single-threaded future executor hosted by an ordinary `ax-task` thread. +#[derive(Clone)] +pub struct LocalExecutor { + core: LocalExecutorCore, + wait: Arc, +} + +impl LocalExecutor { + /// Creates an empty local executor. + pub fn new() -> Self { + let wait = Arc::new(WaitQueue::new()); + let wait_for_pend = wait.clone(); + let host_waker = crate::try_current_task_waker(); + let core = LocalExecutorCore::new(Arc::new(move || { + if let Some(waker) = &host_waker { + let _ = waker.wake(0); + } + wait_for_pend.notify_one(true); + })); + Self { core, wait } + } + + /// Returns a spawner tied to this executor. + pub fn spawner(&self) -> LocalSpawner { + LocalSpawner { + core: self.core.spawner(), + } + } + + /// Polls all ready tasks until no ready task remains. + pub fn run_until_idle(&self) { + self.run_ready_tasks(); + } + + /// Runs ready tasks, blocking the host thread when all local tasks are + /// pending and `external_ready` is false. + /// + /// The function returns once no local task is ready and `external_ready` + /// evaluates true. Runtime users normally drain the external event and call + /// this again from their worker loop. + #[track_caller] + pub fn run_until_idle_with(&self, external_ready: impl Fn() -> bool) { + self.enter(); + loop { + self.core.poll_ready(); + if external_ready() || !self.has_live_tasks() { + self.leave(); + return; + } + self.wait + .wait_until(|| self.core.has_ready_tasks() || external_ready()); + } + } + + /// Runs ready tasks and sleeps on a [`RuntimeEvent`] when all local tasks + /// are pending. + /// + /// When the runtime event is published from IRQ context, this method returns + /// to task context first, wakes futures registered on the event, and then + /// polls the ready queue. This keeps hard IRQ callbacks from invoking + /// arbitrary `Waker` implementations. + #[track_caller] + pub fn run_until_event(&self, event: &RuntimeEvent, external_ready: impl Fn() -> bool) { + self.enter(); + loop { + self.core.poll_ready(); + if external_ready() { + event.wake_waiters_deferred(); + if self.core.has_ready_tasks() { + continue; + } + self.leave(); + return; + } + if !self.has_live_tasks() { + self.leave(); + return; + } + event.wait_until(|| self.core.has_ready_tasks() || external_ready()); + event.wake_waiters_deferred(); + } + } + + fn run_ready_tasks(&self) { + self.enter(); + self.core.poll_ready(); + self.leave(); + } + + fn enter(&self) { + self.core.enter(); + } + + fn leave(&self) { + self.core.leave(); + } + + fn has_live_tasks(&self) -> bool { + self.core.has_live_tasks() + } +} + +impl Default for LocalExecutor { + fn default() -> Self { + Self::new() + } +} + +/// Handle used to spawn futures onto a [`LocalExecutor`]. +#[derive(Clone)] +pub struct LocalSpawner { + core: LocalSpawnerCore, +} + +impl LocalSpawner { + /// Spawns a future on the local executor. + pub fn spawn_local(&self, future: F) -> Result<(), SpawnLocalError> + where + F: Future + 'static, + { + self.core.spawn_local(future) + } +} diff --git a/os/arceos/modules/axtask/src/run_queue.rs b/os/arceos/modules/axtask/src/run_queue.rs index a1daea1383..740246d966 100644 --- a/os/arceos/modules/axtask/src/run_queue.rs +++ b/os/arceos/modules/axtask/src/run_queue.rs @@ -1,16 +1,24 @@ +#[cfg(all(test, feature = "host-test", feature = "smp"))] +use alloc::boxed::Box; use alloc::{collections::VecDeque, sync::Arc}; use core::mem::MaybeUninit; #[cfg(feature = "smp")] use core::ptr::NonNull; -#[cfg(all(feature = "smp", feature = "ipi"))] +#[cfg(all(test, feature = "host-test", feature = "smp"))] use core::sync::atomic::{AtomicBool, Ordering}; use ax_hal::percpu::this_cpu_id; use ax_kernel_guard::BaseGuard; -use ax_kspin::{SpinNoIrqGuard, SpinRaw}; +use ax_kspin::SpinNoIrq; use ax_lazyinit::LazyInit; use ax_memory_addr::VirtAddr; -use ax_sched::BaseScheduler; +#[cfg(all( + feature = "smp", + any(feature = "ipi", feature = "irq-wake-ipi"), + not(all(test, feature = "host-test")) +))] +use bare_task::IpiEvent; +use bare_task::{BareCpuCore, CpuId, WaitQueueCore}; use crate::{ AxCpuMask, AxTaskRef, Scheduler, TaskInner, WaitQueue, @@ -58,6 +66,20 @@ static mut RUN_QUEUES: [MaybeUninit<&'static mut AxRunQueue>; crate::build_info: #[allow(clippy::declare_interior_mutable_const)] // It's ok because it's used only for initialization `RUN_QUEUES`. const ARRAY_REPEAT_VALUE: MaybeUninit<&'static mut AxRunQueue> = MaybeUninit::uninit(); +#[cfg(all(test, feature = "host-test", feature = "smp"))] +static TEST_RUN_QUEUE_INIT: [AtomicBool; crate::build_info::CPU_CAPACITY] = + [const { AtomicBool::new(false) }; crate::build_info::CPU_CAPACITY]; + +#[cfg(all(test, feature = "host-test", feature = "smp"))] +static REMOTE_WAKE_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +#[cfg(all(test, feature = "host-test", feature = "smp"))] +pub(crate) fn remote_wake_test_guard() -> std::sync::MutexGuard<'static, ()> { + REMOTE_WAKE_TEST_LOCK + .lock() + .unwrap_or_else(|err| err.into_inner()) +} + #[cfg(not(feature = "host-test"))] fn main_task_stack() -> TaskStack { let (stack_ptr, stack_size) = ax_hal::mem::boot_stack_bounds(this_cpu_id()); @@ -164,30 +186,52 @@ fn request_current_reschedule() { } } -#[cfg(all(test, feature = "smp", feature = "ipi", feature = "host-test"))] -static REMOTE_RESCHEDULE_REQUESTS: core::sync::atomic::AtomicUsize = - core::sync::atomic::AtomicUsize::new(0); - #[cfg(all( + test, feature = "smp", - feature = "ipi", - not(all(test, feature = "host-test")) + any(feature = "ipi", feature = "irq-wake-ipi"), + feature = "host-test" ))] -static REMOTE_RESCHEDULE_PENDING: [AtomicBool; crate::build_info::CPU_CAPACITY] = - [const { AtomicBool::new(false) }; crate::build_info::CPU_CAPACITY]; +static REMOTE_RESCHEDULE_REQUESTS: core::sync::atomic::AtomicUsize = + core::sync::atomic::AtomicUsize::new(0); #[cfg(all(test, feature = "smp", feature = "ipi", feature = "host-test"))] static REMOTE_RESCHEDULE_PENDING: AtomicBool = AtomicBool::new(false); +#[cfg(all( + test, + feature = "smp", + any(feature = "ipi", feature = "irq-wake-ipi"), + feature = "host-test" +))] +static REMOTE_IRQ_WAKE_PENDING: AtomicBool = AtomicBool::new(false); + #[cfg(all(feature = "smp", feature = "ipi"))] pub(crate) fn clear_remote_reschedule_pending_for_current_cpu() { #[cfg(not(all(test, feature = "host-test")))] - REMOTE_RESCHEDULE_PENDING[this_cpu_id()].store(false, Ordering::Release); + get_run_queue(this_cpu_id()) + .core + .clear_ipi(IpiEvent::Reschedule); #[cfg(all(test, feature = "host-test"))] REMOTE_RESCHEDULE_PENDING.store(false, Ordering::Release); } -#[cfg(all(feature = "smp", feature = "ipi"))] +#[cfg(all(feature = "smp", any(feature = "ipi", feature = "irq-wake-ipi")))] +pub(crate) fn clear_remote_irq_wake_pending_for_current_cpu() { + #[cfg(not(all(test, feature = "host-test")))] + get_run_queue(this_cpu_id()) + .core + .clear_ipi(IpiEvent::IrqWakeDrain); + #[cfg(all(test, feature = "host-test"))] + REMOTE_IRQ_WAKE_PENDING.store(false, Ordering::Release); +} + +#[cfg(all( + test, + feature = "smp", + any(feature = "ipi", feature = "irq-wake-ipi"), + feature = "host-test" +))] fn request_remote_reschedule_if_not_pending(pending: &AtomicBool, request: F) where F: FnOnce(), @@ -197,7 +241,7 @@ where } } -#[cfg(all(feature = "smp", feature = "ipi"))] +#[cfg(all(test, feature = "smp", feature = "ipi", feature = "host-test"))] fn force_remote_reschedule_request(pending: &AtomicBool, request: F) where F: FnOnce(), @@ -212,9 +256,9 @@ where not(all(test, feature = "host-test")) ))] fn request_remote_reschedule(cpu_id: usize) { - request_remote_reschedule_if_not_pending(&REMOTE_RESCHEDULE_PENDING[cpu_id], || { + if get_run_queue(cpu_id).core.request_ipi(IpiEvent::Reschedule) { ax_ipi::run_on_cpu(cpu_id, request_current_reschedule); - }); + } } #[cfg(all( @@ -223,9 +267,25 @@ fn request_remote_reschedule(cpu_id: usize) { not(all(test, feature = "host-test")) ))] fn force_remote_reschedule(cpu_id: usize) { - force_remote_reschedule_request(&REMOTE_RESCHEDULE_PENDING[cpu_id], || { - ax_ipi::run_on_cpu(cpu_id, request_current_reschedule); - }); + let _ = get_run_queue(cpu_id).core.request_ipi(IpiEvent::Reschedule); + ax_ipi::run_on_cpu(cpu_id, request_current_reschedule); +} + +#[cfg(all( + feature = "smp", + any(feature = "ipi", feature = "irq-wake-ipi"), + not(all(test, feature = "host-test")) +))] +fn request_remote_irq_wake(cpu_id: usize) { + if get_run_queue(cpu_id) + .core + .request_ipi(IpiEvent::IrqWakeDrain) + { + ax_hal::irq::send_ipi( + ax_hal::irq::ipi_irq(), + ax_hal::irq::IpiTarget::Other { cpu_id }, + ); + } } #[cfg(all(test, feature = "smp", feature = "ipi", feature = "host-test"))] @@ -246,6 +306,19 @@ fn force_remote_reschedule(cpu_id: usize) { }); } +#[cfg(all( + test, + feature = "smp", + any(feature = "ipi", feature = "irq-wake-ipi"), + feature = "host-test" +))] +fn request_remote_irq_wake(cpu_id: usize) { + let _ = cpu_id; + request_remote_reschedule_if_not_pending(&REMOTE_IRQ_WAKE_PENDING, || { + REMOTE_RESCHEDULE_REQUESTS.fetch_add(1, Ordering::Release); + }); +} + #[cfg(all(feature = "smp", feature = "ipi"))] fn kick_remote_cpu(cpu_id: usize) { if cpu_id != this_cpu_id() { @@ -256,6 +329,13 @@ fn kick_remote_cpu(cpu_id: usize) { } } +#[cfg(all(feature = "smp", any(feature = "ipi", feature = "irq-wake-ipi")))] +pub(crate) fn kick_remote_cpu_for_irq_wake(cpu_id: usize) { + if cpu_id != this_cpu_id() { + request_remote_irq_wake(cpu_id); + } +} + #[cfg(all(feature = "smp", feature = "ipi"))] fn force_kick_remote_cpu(cpu_id: usize) { if cpu_id != this_cpu_id() { @@ -271,10 +351,12 @@ mod tests { // keep the shared pending/count assertions in one test. #[test] fn remote_reschedule_request_is_coalesced_and_forced() { + let _remote_wake_guard = super::remote_wake_test_guard(); const REMOTE_CPU: usize = 1; super::REMOTE_RESCHEDULE_REQUESTS.store(0, Ordering::Release); super::REMOTE_RESCHEDULE_PENDING.store(false, Ordering::Release); + super::REMOTE_IRQ_WAKE_PENDING.store(false, Ordering::Release); super::kick_remote_cpu(REMOTE_CPU); @@ -338,6 +420,7 @@ mod tests { } super::REMOTE_RESCHEDULE_PENDING.store(false, Ordering::Release); + super::REMOTE_IRQ_WAKE_PENDING.store(false, Ordering::Release); super::REMOTE_RESCHEDULE_REQUESTS.store(0, Ordering::Release); } @@ -376,6 +459,29 @@ mod tests { "forced remote kicks must not coalesce required migration reschedules", ); } + + #[test] + fn remote_irq_wake_does_not_coalesce_scheduler_reschedule() { + let _remote_wake_guard = super::remote_wake_test_guard(); + const REMOTE_CPU: usize = 1; + + super::REMOTE_RESCHEDULE_REQUESTS.store(0, Ordering::Release); + super::REMOTE_RESCHEDULE_PENDING.store(false, Ordering::Release); + super::REMOTE_IRQ_WAKE_PENDING.store(false, Ordering::Release); + + super::kick_remote_cpu_for_irq_wake(REMOTE_CPU); + super::kick_remote_cpu(REMOTE_CPU); + + assert_eq!( + super::REMOTE_RESCHEDULE_REQUESTS.load(Ordering::Acquire), + 2, + "a pending IRQ-wake IPI must not suppress a scheduler-visible reschedule callback", + ); + + super::REMOTE_RESCHEDULE_PENDING.store(false, Ordering::Release); + super::REMOTE_IRQ_WAKE_PENDING.store(false, Ordering::Release); + super::REMOTE_RESCHEDULE_REQUESTS.store(0, Ordering::Release); + } } /// Selects the appropriate run queue for the provided task. @@ -471,7 +577,7 @@ pub(crate) struct AxRunQueue { /// The core scheduler of this run queue. /// Since irq and preempt are preserved by the kernel guard hold by `AxRunQueueRef`, /// we just use a simple raw spin lock here. - scheduler: SpinRaw, + core: BareCpuCore, } /// A reference to the run queue with specific guard. @@ -522,7 +628,7 @@ impl AxRunQueueRef<'_, G> { assert!(task.is_ready()); #[cfg(feature = "smp")] task.set_cpu_id(cpu_id as _); - self.inner.scheduler.lock().add_task(task); + self.inner.core.with_run_queue(|core| core.add_task(task)); #[cfg(all(feature = "smp", feature = "ipi"))] kick_remote_cpu(cpu_id); } @@ -531,7 +637,7 @@ impl AxRunQueueRef<'_, G> { /// /// This function does nothing if the task is not in [`TaskState::Blocked`], /// which means the task is already unblocked by other cores. - pub fn unblock_task(&mut self, task: AxTaskRef, resched: bool) { + pub fn unblock_task(&mut self, task: AxTaskRef, resched: bool) -> bool { let task_id_name = if log::log_enabled!(log::Level::Debug) { Some(task.id_name()) } else { @@ -559,41 +665,26 @@ impl AxRunQueueRef<'_, G> { } #[cfg(all(feature = "smp", feature = "ipi"))] kick_remote_cpu(cpu_id); + true + } else { + false } } } /// Core functions of run queue. impl CurrentRunQueueRef<'_, G> { - /// Unblock one task by inserting it into the current CPU's run queue. - /// - /// See [`AxRunQueueRef::unblock_task`] for the state-transition details. - #[cfg(feature = "irq")] - pub(crate) fn unblock_task(&mut self, task: AxTaskRef, resched: bool) { - let task_id_name = if log::log_enabled!(log::Level::Debug) { - Some(task.id_name()) - } else { - None - }; - if self - .inner - .put_task_with_state(task, TaskState::Blocked, resched) - { - let cpu_id = self.inner.cpu_id; - if let Some(task_id_name) = task_id_name { - debug!("task unblock: {task_id_name} on run_queue {cpu_id}"); - } - if resched { - #[cfg(feature = "preempt")] - crate::current().set_preempt_pending(true); - } - } + pub(crate) fn mark_timer_irq_pending(&mut self) { + self.inner.core.mark_timer_service_pending(); + } + + pub(crate) fn clear_timer_service_pending(&mut self) { + self.inner.core.clear_timer_service_pending(); } - #[cfg(feature = "irq")] pub fn scheduler_timer_tick(&mut self) { let curr = &self.current_task; - if !curr.is_idle() && self.inner.scheduler.lock().task_tick(curr) { + if !curr.is_idle() && self.inner.core.with_run_queue(|core| core.task_tick(curr)) { #[cfg(feature = "preempt")] curr.set_preempt_pending(true); } @@ -760,13 +851,14 @@ impl CurrentRunQueueRef<'_, G> { } /// Block the current task, put current task into the wait queue and reschedule. - /// Mark the state of current task as `Blocked`, set the `in_wait_queue` flag as true. + /// Mark the state of current task as `Blocked` and record this queue as + /// the task's current wait-queue membership. /// Note: /// 1. The caller must hold the lock of the wait queue. /// 2. The caller must ensure that the current task is in the running state. /// 3. The caller must ensure that the current task is not the idle task. /// 4. The lock of the wait queue will be released explicitly after current task is pushed into it. - pub fn blocked_resched(&mut self, mut wq_guard: WaitQueueGuard) { + pub fn blocked_resched(&mut self, mut wq_guard: WaitQueueGuard, wait_queue_key: usize) { let curr = &self.current_task; assert!(curr.is_running()); assert!(!curr.is_idle()); @@ -780,13 +872,12 @@ impl CurrentRunQueueRef<'_, G> { // while holding the lock of the wait queue. curr.set_state(TaskState::Blocked); - // A preemptive future wake can re-enter a wait path before a previous - // wait-queue entry has been consumed. Avoid leaving a stale duplicate - // waiter that may receive mutex ownership after the task is running. - if !curr.in_wait_queue() { - curr.set_in_wait_queue(true); - wq_guard.push_back(curr.clone()); - } + // Wait-queue membership is keyed by the queue address. If the task was + // left in an older queue, that entry becomes stale and the old queue will + // discard it without clearing this new membership. + wq_guard.retain(|task| !curr.ptr_eq(task)); + curr.set_wait_queue_key(wait_queue_key); + wq_guard.push_back(curr.clone()); // Drop the lock of wait queue explicitly. drop(wq_guard); @@ -798,22 +889,68 @@ impl CurrentRunQueueRef<'_, G> { self.inner.resched(); } + /// Block the current task in `wq_guard`, then optionally abort before the + /// actual context switch. + /// + /// The abort predicate runs after the task has been linked into the wait + /// queue, but after the wait-queue lock is dropped. If it aborts the sleep, + /// the task is removed from the same wait queue before returning to keep + /// wait-queue membership and task state in sync. + pub fn blocked_resched_abortable( + &mut self, + wait_queue: &SpinNoIrq>, + should_abort_sleep: impl Fn() -> bool, + ) -> bool { + let curr = &self.current_task; + assert!(curr.is_running()); + assert!(!curr.is_idle()); + // Current expected preempt count is 1 for `NoPreemptIrqSave`. + // This helper takes the wait-queue lock after the check below. + #[cfg(feature = "preempt")] + assert!(curr.can_preempt(1)); + + let wait_queue_key = core::ptr::from_ref(wait_queue) as usize; + let mut wq_guard = wait_queue.lock(); + curr.set_state(TaskState::Blocked); + wq_guard.retain(|task| !curr.ptr_eq(task)); + curr.set_wait_queue_key(wait_queue_key); + wq_guard.push_back(curr.clone()); + drop(wq_guard); + + if should_abort_sleep() { + let mut wq_guard = wait_queue.lock(); + if curr.clear_wait_queue_key(wait_queue_key) { + wq_guard.retain(|task| !curr.ptr_eq(task)); + } + drop(wq_guard); + + if curr.transition_state(TaskState::Blocked, TaskState::Running) { + return true; + } + } + + debug!("task block: {}", curr.id_name()); + self.inner.resched(); + false + } + /// Block the current task, put current task into the wait queue and reschedule. /// This is special just for future. - pub fn future_blocked_resched(&mut self, mut woke: SpinNoIrqGuard<'_, bool>) { + pub fn future_blocked_resched(&mut self, should_abort_sleep: impl Fn() -> bool) { let curr = &self.current_task; assert!(curr.is_running()); assert!(!curr.is_idle()); // we must not block current task with preemption disabled. - // Current expected preempt count is 2 for `NoPreemptIrqSave` and `woke`. + // Current expected preempt count is 1 for `NoPreemptIrqSave`. #[cfg(feature = "preempt")] - assert!(curr.can_preempt(2)); + assert!(curr.can_preempt(1)); // Mark the task as blocked, this has to be done before adding it to the wait queue // while holding the lock of the wait queue. curr.set_state(TaskState::Blocked); - *woke = false; - drop(woke); + if should_abort_sleep() && curr.transition_state(TaskState::Blocked, TaskState::Running) { + return; + } // Current task's state has been changed to `Blocked` and added to the wait queue. // Note that the state may have been set as `Ready` in `unblock_task()`, @@ -823,7 +960,6 @@ impl CurrentRunQueueRef<'_, G> { self.inner.resched(); } - #[cfg(feature = "irq")] pub fn sleep_until(&mut self, deadline: ax_hal::time::TimeValue) { let curr = &self.current_task; debug!("task sleep: {}, deadline={:?}", curr.id_name(), deadline); @@ -839,9 +975,8 @@ impl CurrentRunQueueRef<'_, G> { pub fn set_current_priority(&mut self, prio: isize) -> bool { self.inner - .scheduler - .lock() - .set_priority(&self.current_task, prio) + .core + .with_run_queue(|core| core.set_priority(&self.current_task, prio)) } #[cfg(feature = "smp")] @@ -867,12 +1002,9 @@ impl AxRunQueue { // gc task should be pinned to the current CPU. gc_task.set_cpumask(AxCpuMask::one_shot(cpu_id)); - let mut scheduler = Scheduler::new(); - scheduler.add_task(gc_task); - Self { - cpu_id, - scheduler: SpinRaw::new(scheduler), - } + let core = BareCpuCore::new(CpuId(cpu_id), Scheduler::new()); + core.with_run_queue(|run_queue| run_queue.add_task(gc_task)); + Self { cpu_id, core } } /// Puts target task into current run queue with `Ready` state @@ -888,13 +1020,15 @@ impl AxRunQueue { current_state: TaskState, preempt: bool, ) -> bool { - // If the task's state matches `current_state`, set its state to `Ready` and - // put it back to the run queue (except idle task). - if task.transition_state(current_state, TaskState::Ready) && !task.is_idle() { + // If the task's state matches `current_state`, set it to `Ready` through + // the bare run-queue core and then put it back to the scheduler (except idle task). + if self + .core + .with_run_queue(|core| core.transition_to_ready(&task, current_state)) + { #[cfg(feature = "smp")] - let waking_current_task = current_state == TaskState::Blocked - && self.cpu_id == this_cpu_id() - && crate::current().ptr_eq(&task); + let waking_local_task = + current_state == TaskState::Blocked && self.cpu_id == this_cpu_id(); // If the task is blocked, wait for the task to finish its scheduling process. // See `unblock_task()` for details. if current_state == TaskState::Blocked { @@ -910,10 +1044,12 @@ impl AxRunQueue { // 2. This can be placed in the front of `switch_to()` #[cfg(feature = "smp")] { - // A scheduler tracepoint or other IRQ-safe notification can wake the - // task that is currently being switched out on this CPU. Waiting for - // `on_cpu` there would wait for the very switch we are still inside. - if !waking_current_task { + // A local wake can target the previous task that is still marked + // `on_cpu` until this CPU completes the switch bookkeeping. Waiting + // here would deadlock: the target must be queued before it can run + // again and finish clearing the marker. Remote wakes still wait for + // the owning CPU to finish touching the task context. + if !waking_local_task { while task.on_cpu() { // Wait for the task to finish its scheduling process. core::hint::spin_loop(); @@ -924,7 +1060,8 @@ impl AxRunQueue { // TODO: priority #[cfg(feature = "smp")] task.set_cpu_id(self.cpu_id as _); - self.scheduler.lock().put_prev_task(task, preempt); + self.core + .with_run_queue(|core| core.put_prev_task(task, preempt)); true } else { false @@ -935,9 +1072,8 @@ impl AxRunQueue { /// Pick the next task to run and switch to it. fn resched(&mut self) { let next = self - .scheduler - .lock() - .pick_next_task() + .core + .with_run_queue(|core| core.pick_next_task()) .unwrap_or_else(|| unsafe { // Safety: IRQs must be disabled at this time. IDLE_TASK.current_ref_raw().get_unchecked().clone() @@ -953,7 +1089,7 @@ impl AxRunQueue { fn switch_to(&mut self, prev_task: CurrentTask, next_task: AxTaskRef) { // Make sure that IRQs are disabled by kernel guard or other means. - #[cfg(all(feature = "irq", not(feature = "host-test")))] + #[cfg(not(feature = "host-test"))] assert!( !ax_hal::asm::irqs_enabled(), "IRQs must be disabled during scheduling" @@ -1002,11 +1138,15 @@ impl AxRunQueue { ); unsafe { + // `CurrentTask` is a borrowed Arc view over the per-CPU current-task + // pointer. `set_current()` consumes that per-CPU strong reference, + // so keep an explicit owner alive until this saved context resumes. + let _prev_task_keepalive = prev_task.clone(); let prev_ctx_ptr = prev_task.ctx_mut_ptr(); let next_ctx_ptr = next_task.ctx_mut_ptr(); // Store a raw pointer to prev_task in PREV_TASK. - // Safety: prev_task is alive (Arc held on caller's stack) and will + // Safety: prev_task is alive (`_prev_task_keepalive`) and will // remain so through clear_prev_task_on_cpu() below. #[cfg(feature = "smp")] { @@ -1056,16 +1196,11 @@ fn gc_entry() { // Note: we cannot block current task with preemption disabled, // use `current_ref_raw` to get the `WAIT_FOR_EXIT`'s reference here to avoid the use of `NoPreemptGuard`. // Since gc task is pinned to the current CPU, there is no effect if the gc task is preempted during the process. - #[cfg(feature = "irq")] unsafe { let _timeout = WAIT_FOR_EXIT .current_ref_raw() .wait_timeout(core::time::Duration::from_millis(100)); } - #[cfg(not(feature = "irq"))] - unsafe { - WAIT_FOR_EXIT.current_ref_raw().wait(); - } } } @@ -1079,9 +1214,8 @@ pub(crate) fn migrate_entry(migrated_task: AxTaskRef) { let cpu_id = rq.inner.cpu_id; migrated_task.set_cpu_id(cpu_id as _); rq.inner - .scheduler - .lock() - .put_prev_task(migrated_task, false); + .core + .with_run_queue(|core| core.put_prev_task(migrated_task, false)); #[cfg(all(feature = "smp", feature = "ipi"))] // Current-task migration cannot make progress until the target CPU runs // the migrated task, so do not let a stale coalescing bit suppress this IPI. @@ -1098,6 +1232,23 @@ pub(crate) unsafe fn clear_prev_task_on_cpu() { // (switch_to has not yet returned), so the pointer is valid. unsafe { prev.as_ref() }.set_on_cpu(false); } + +pub(crate) fn wake_task_from_irq_queue(task: AxTaskRef) -> bool { + select_wake_run_queue::(&task).unblock_task(task, true) +} + +#[cfg(all(test, feature = "host-test", feature = "smp"))] +pub(crate) fn init_test_run_queue_for_cpu(cpu_id: usize) { + assert!(cpu_id < crate::build_info::CPU_CAPACITY); + if TEST_RUN_QUEUE_INIT[cpu_id].swap(true, Ordering::AcqRel) { + return; + } + let run_queue = Box::leak(Box::new(AxRunQueue::new(cpu_id))); + unsafe { + RUN_QUEUES[cpu_id].write(run_queue); + } +} + pub(crate) fn init() { let cpu_id = this_cpu_id(); @@ -1123,6 +1274,7 @@ pub(crate) fn init() { RUN_QUEUE.with_current(|rq| { rq.init_once(AxRunQueue::new(cpu_id)); }); + crate::irq_wake::init_irq_wake_queue_current_cpu(); unsafe { RUN_QUEUES[cpu_id].write(RUN_QUEUE.current_ref_mut_raw()); } @@ -1146,6 +1298,7 @@ pub(crate) fn init_secondary(stack_ptr: VirtAddr, stack_size: usize) { RUN_QUEUE.with_current(|rq| { rq.init_once(AxRunQueue::new(cpu_id)); }); + crate::irq_wake::init_irq_wake_queue_current_cpu(); unsafe { RUN_QUEUES[cpu_id].write(RUN_QUEUE.current_ref_mut_raw()); } diff --git a/os/arceos/modules/axtask/src/task.rs b/os/arceos/modules/axtask/src/task.rs index ff15ec8c0d..a19d960bec 100644 --- a/os/arceos/modules/axtask/src/task.rs +++ b/os/arceos/modules/axtask/src/task.rs @@ -1,17 +1,14 @@ use alloc::{boxed::Box, string::String, sync::Arc}; #[cfg(not(feature = "stack-guard-page"))] use core::alloc::Layout; -#[cfg(any( - feature = "preempt", - all(feature = "stack-guard-page", feature = "smp", feature = "ipi") -))] +#[cfg(all(feature = "stack-guard-page", feature = "smp", feature = "ipi"))] use core::sync::atomic::AtomicUsize; use core::{ cell::{Cell, UnsafeCell}, fmt, mem::ManuallyDrop, ops::Deref, - sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicU32, AtomicU64, Ordering}, + sync::atomic::{AtomicI32, Ordering}, task::{Context, Poll}, }; @@ -22,6 +19,7 @@ use ax_kspin::SpinNoIrq; #[cfg(feature = "stack-guard-page")] use ax_memory_addr::PAGE_SIZE_4K; use ax_memory_addr::{VirtAddr, align_up_4k}; +use bare_task::{CpuId, CpuMask, TaskCore}; use futures_util::task::AtomicWaker; #[cfg(feature = "lockdep")] @@ -38,23 +36,10 @@ const STACK_END_MAGIC: usize = 0x57AC_CE11usize; pub(crate) const TASK_STACK_ALIGN: usize = 16; /// A unique identifier for a thread. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub struct TaskId(u64); +pub type TaskId = bare_task::TaskId; /// The possible states of a task. -#[repr(u8)] -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum TaskState { - /// Task is running on some CPU. - Running = 1, - /// Task is ready to run on some scheduler's ready queue. - Ready = 2, - /// Task is blocked (in the wait queue or timer list), - /// and it has finished its scheduling process, it can be wake up by `notify()` on any run queue safely. - Blocked = 3, - /// Task is exited and waiting for being dropped. - Exited = 4, -} +pub type TaskState = bare_task::TaskState; /// User-defined task extended data. #[cfg(feature = "task-ext")] @@ -71,16 +56,12 @@ pub trait TaskExt { /// The inner task structure. pub struct TaskInner { - id: TaskId, + core: TaskCore, name: SpinNoIrq, is_idle: bool, is_init: bool, entry: Cell>>, - state: AtomicU8, - - /// CPU affinity mask. - cpumask: SpinNoIrq, /// Scheduling policy of the task. sched_policy: AtomicI32, @@ -88,29 +69,6 @@ pub struct TaskInner { /// Scheduling priority of the task. sched_priority: AtomicI32, - /// Mark whether the task is in the wait queue. - in_wait_queue: AtomicBool, - - /// Used to indicate the CPU ID where the task is running or will run. - cpu_id: AtomicU32, - /// Used to indicate whether the task is running on a CPU. - #[cfg(feature = "smp")] - on_cpu: AtomicBool, - - /// A ticket ID used to identify the timer event. - /// Set by `set_timer_ticket()` when creating a timer event in `set_alarm_wakeup()`, - /// expired by setting it as zero in `timer_ticket_expired()`, which is called by `cancel_events()`. - #[cfg(feature = "irq")] - timer_ticket_id: AtomicU64, - - #[cfg(feature = "preempt")] - need_resched: AtomicBool, - #[cfg(feature = "preempt")] - force_resched: AtomicBool, - #[cfg(feature = "preempt")] - preempt_disable_count: AtomicUsize, - - interrupted: AtomicBool, interrupt_waker: AtomicWaker, exit_code: AtomicI32, @@ -128,31 +86,6 @@ pub struct TaskInner { tls: TlsArea, } -impl TaskId { - fn new() -> Self { - static ID_COUNTER: AtomicU64 = AtomicU64::new(1); - Self(ID_COUNTER.fetch_add(1, Ordering::Relaxed)) - } - - /// Convert the task ID to a `u64`. - pub const fn as_u64(&self) -> u64 { - self.0 - } -} - -impl From for TaskState { - #[inline] - fn from(state: u8) -> Self { - match state { - 1 => Self::Running, - 2 => Self::Ready, - 3 => Self::Blocked, - 4 => Self::Exited, - _ => unreachable!(), - } - } -} - unsafe impl Send for TaskInner {} unsafe impl Sync for TaskInner {} @@ -163,7 +96,7 @@ impl TaskInner { F: FnOnce() + Send + 'static, { let kstack = TaskStack::alloc(align_up_4k(stack_size)); - let mut t = Self::new_common(TaskId::new(), name, kstack); + let mut t = Self::new_common(TaskId::allocate(), name, kstack); debug!("new task: {}", t.id_name()); #[cfg(feature = "tls")] @@ -183,7 +116,7 @@ impl TaskInner { /// Gets the ID of the task. pub const fn id(&self) -> TaskId { - self.id + self.core.id() } /// Gets the name of the task. @@ -198,7 +131,7 @@ impl TaskInner { /// Get a combined string of the task ID and name. pub fn id_name(&self) -> alloc::string::String { - alloc::format!("Task({}, {:?})", self.id.as_u64(), self.name()) + alloc::format!("Task({}, {:?})", self.id().as_u64(), self.name()) } /// Wait for the task to exit, and return the exit code. @@ -253,7 +186,7 @@ impl TaskInner { /// Note: the task may not be running on the CPU, it just exists in the run queue. #[inline] pub fn cpu_id(&self) -> u32 { - self.cpu_id.load(Ordering::Acquire) + self.core.cpu_id().0 as u32 } /// Gets the cpu affinity mask of the task. @@ -261,7 +194,7 @@ impl TaskInner { /// Returns the cpu affinity mask of the task in type [`AxCpuMask`]. #[inline] pub fn cpumask(&self) -> AxCpuMask { - *self.cpumask.lock() + bare_to_ax_cpumask(self.core.cpumask()) } /// Sets the cpu affinity mask of the task. @@ -270,7 +203,7 @@ impl TaskInner { /// `cpumask` - The cpu affinity mask to be set in type [`AxCpuMask`]. #[inline] pub fn set_cpumask(&self, cpumask: AxCpuMask) { - *self.cpumask.lock() = cpumask + self.core.set_cpumask(ax_to_bare_cpumask(cpumask)); } #[inline] @@ -301,7 +234,7 @@ impl TaskInner { // allow `interrupt()` to run and call `wake()` on an empty waker // slot — the wake is lost. Registering first closes the window. self.interrupt_waker.register(cx.waker()); - if self.interrupted.swap(false, Ordering::AcqRel) { + if self.core.take_interrupt() { Poll::Ready(()) } else { Poll::Pending @@ -311,7 +244,7 @@ impl TaskInner { /// Clears the interrupt state of the task. #[inline] pub fn clear_interrupt(&self) { - self.interrupted.store(false, Ordering::Release); + self.core.clear_interrupt(); } /// Atomically checks and clears the interrupt flag. @@ -319,7 +252,7 @@ impl TaskInner { /// Returns `true` if the task was interrupted. #[inline] pub fn take_interrupt(&self) -> bool { - self.interrupted.swap(false, Ordering::AcqRel) + self.core.take_interrupt() } /// Checks whether the task has been interrupted without clearing @@ -330,44 +263,94 @@ impl TaskInner { /// consumers (e.g., an [`interruptible`] future wrapper). #[inline] pub fn interrupted(&self) -> bool { - self.interrupted.load(Ordering::Acquire) + self.core.interrupted() } /// Interrupts the task. #[inline] pub fn interrupt(&self) { - self.interrupted.store(true, Ordering::Release); + self.core.interrupt(); self.interrupt_waker.wake(); } + + #[inline] + pub(crate) fn irq_wake_generation(&self) -> u64 { + self.core.wake_generation() + } + + #[inline] + pub(crate) fn irq_wake_generation_matches(&self, generation: u64) -> bool { + self.core.wake_generation_matches(generation) + } + + #[inline] + pub(crate) fn expire_irq_wakers(&self) { + self.core.expire_wakers(); + } + + #[inline] + pub(crate) fn publish_irq_wake_bits(&self, bits: u64) { + if bits != 0 { + self.core.publish_wake_bits(bits); + } + } + + #[inline] + pub(crate) fn take_irq_wake_bits(&self) -> u64 { + self.core.take_wake_bits() + } + + #[inline] + pub(crate) fn bump_irq_wake_seq(&self) -> u64 { + self.core.bump_wake_seq() + } + + #[inline] + pub(crate) fn irq_wake_seq(&self) -> u64 { + self.core.wake_seq() + } + + #[inline] + pub(crate) fn mark_irq_wake_pending(&self) -> bool { + self.core.mark_wake_pending() + } + + #[inline] + pub(crate) fn take_irq_wake_pending(&self) -> bool { + self.core.take_wake_pending() + } + + #[inline] + pub(crate) fn set_irq_wake_next(&self, next: *mut AxTask) { + self.core.set_wake_next(next); + } + + #[inline] + pub(crate) fn irq_wake_next(&self) -> *mut AxTask { + self.core.wake_next() + } + + #[inline] + pub(crate) fn clear_irq_wake_link(&self) { + self.core.clear_wake_link(); + } } // private methods impl TaskInner { fn new_common(id: TaskId, name: String, kstack: TaskStack) -> Self { Self { - id, + core: { + let core = TaskCore::new(id, CpuId(0)); + core.set_cpumask(ax_to_bare_cpumask(crate::api::cpu_mask_full())); + core + }, name: SpinNoIrq::new(name), is_idle: false, is_init: false, entry: Cell::new(None), - state: AtomicU8::new(TaskState::Ready as u8), - // By default, the task is allowed to run on all CPUs. - cpumask: SpinNoIrq::new(crate::api::cpu_mask_full()), sched_policy: AtomicI32::new(0), sched_priority: AtomicI32::new(0), - in_wait_queue: AtomicBool::new(false), - #[cfg(feature = "irq")] - timer_ticket_id: AtomicU64::new(0), - cpu_id: AtomicU32::new(0), - #[cfg(feature = "smp")] - on_cpu: AtomicBool::new(false), - #[cfg(feature = "preempt")] - need_resched: AtomicBool::new(false), - #[cfg(feature = "preempt")] - force_resched: AtomicBool::new(false), - #[cfg(feature = "preempt")] - preempt_disable_count: AtomicUsize::new(0), - interrupted: AtomicBool::new(false), interrupt_waker: AtomicWaker::new(), exit_code: AtomicI32::new(0), wait_for_exit: WaitQueue::new(), @@ -391,9 +374,11 @@ impl TaskInner { /// And there is no need to set the `entry`, `kstack` or `tls` fields, as /// they will be filled automatically when the task is switches out. pub(crate) fn new_init(name: String, kstack: TaskStack) -> Self { - let mut t = Self::new_common(TaskId::new(), name, kstack); + let mut t = Self::new_common(TaskId::allocate(), name, kstack); t.is_init = true; #[cfg(feature = "smp")] + t.set_cpu_id(ax_hal::percpu::this_cpu_id() as u32); + #[cfg(feature = "smp")] t.set_on_cpu(true); if t.name() == "idle" { t.is_idle = true; @@ -408,12 +393,12 @@ impl TaskInner { /// Returns the current state of the task. #[inline] pub fn state(&self) -> TaskState { - self.state.load(Ordering::Acquire).into() + self.core.state() } #[inline] pub(crate) fn set_state(&self, state: TaskState) { - self.state.store(state as u8, Ordering::Release) + self.core.set_state(state) } /// Transition the task state from `current_state` to `new_state`, @@ -421,14 +406,7 @@ impl TaskInner { /// otherwise returns `false`. #[inline] pub(crate) fn transition_state(&self, current_state: TaskState, new_state: TaskState) -> bool { - self.state - .compare_exchange( - current_state as u8, - new_state as u8, - Ordering::AcqRel, - Ordering::Acquire, - ) - .is_ok() + self.core.transition_state(current_state, new_state) } #[inline] @@ -453,66 +431,76 @@ impl TaskInner { #[inline] pub(crate) fn in_wait_queue(&self) -> bool { - self.in_wait_queue.load(Ordering::Acquire) + self.core.in_wait_queue() + } + + #[inline] + pub(crate) fn is_waiting_on(&self, wait_queue_key: usize) -> bool { + self.core.is_waiting_on(wait_queue_key) + } + + #[inline] + pub(crate) fn set_wait_queue_key(&self, wait_queue_key: usize) { + self.core.set_wait_queue_key(wait_queue_key); } #[inline] - pub(crate) fn set_in_wait_queue(&self, in_wait_queue: bool) { - self.in_wait_queue.store(in_wait_queue, Ordering::Release); + pub(crate) fn clear_wait_queue_key(&self, wait_queue_key: usize) -> bool { + self.core.clear_wait_queue_key(wait_queue_key) + } + + #[inline] + pub(crate) fn clear_wait_queue_membership(&self) -> bool { + self.core.take_wait_queue_key() != 0 } /// Returns task's current timer ticket ID. #[inline] - #[cfg(feature = "irq")] pub(crate) fn timer_ticket(&self) -> u64 { - self.timer_ticket_id.load(Ordering::Acquire) + self.core.timer_ticket() } /// Set the timer ticket ID. #[inline] - #[cfg(feature = "irq")] pub(crate) fn set_timer_ticket(&self, timer_ticket_id: u64) { // CAN NOT set timer_ticket_id to 0, // because 0 is used to indicate the timer event is expired. assert!(timer_ticket_id != 0); - self.timer_ticket_id - .store(timer_ticket_id, Ordering::Release); + self.core.set_timer_ticket(timer_ticket_id); } /// Expire timer ticket ID by setting it to 0, /// it can be used to identify one timer event is triggered or expired. - #[inline] - #[cfg(feature = "irq")] pub(crate) fn timer_ticket_expired(&self) { - self.timer_ticket_id.store(0, Ordering::Release); + self.core.timer_ticket_expired(); } #[inline] #[cfg(feature = "preempt")] pub(crate) fn set_preempt_pending(&self, pending: bool) { - self.need_resched.store(pending, Ordering::Release) + self.core.set_preempt_pending(pending) } #[inline] #[cfg(feature = "preempt")] pub(crate) fn set_force_resched_pending(&self, pending: bool) { - self.force_resched.store(pending, Ordering::Release) + self.core.set_force_resched_pending(pending) } #[inline] #[cfg(feature = "preempt")] fn force_resched_pending(&self) -> bool { - self.force_resched.load(Ordering::Acquire) + self.core.force_resched_pending() } #[inline] - #[cfg(all(test, feature = "preempt"))] + #[cfg(all(test, feature = "preempt", feature = "ipi"))] pub(crate) fn preempt_pending_for_test(&self) -> bool { - self.need_resched.load(Ordering::Acquire) + self.core.preempt_pending() } #[inline] - #[cfg(all(test, feature = "preempt"))] + #[cfg(all(test, feature = "preempt", feature = "ipi"))] pub(crate) fn force_resched_pending_for_test(&self) -> bool { self.force_resched_pending() } @@ -520,31 +508,31 @@ impl TaskInner { #[inline] #[cfg(feature = "preempt")] fn take_force_resched_pending(&self) -> bool { - self.force_resched.swap(false, Ordering::AcqRel) + self.core.take_force_resched_pending() } #[inline] #[cfg(feature = "preempt")] pub(crate) fn preempt_count(&self) -> usize { - self.preempt_disable_count.load(Ordering::Acquire) + self.core.preempt_count() } #[inline] #[cfg(feature = "preempt")] pub(crate) fn can_preempt(&self, current_disable_count: usize) -> bool { - self.preempt_disable_count.load(Ordering::Acquire) == current_disable_count + self.core.can_preempt(current_disable_count) } #[inline] #[cfg(feature = "preempt")] pub(crate) fn disable_preempt(&self) { - self.preempt_disable_count.fetch_add(1, Ordering::Release); + self.core.disable_preempt(); } #[inline] #[cfg(feature = "preempt")] pub(crate) fn enable_preempt(&self, resched: bool) { - if self.preempt_disable_count.fetch_sub(1, Ordering::Release) == 1 && resched { + if self.core.enable_preempt() && resched { // If current task is pending to be preempted, do rescheduling. Self::current_check_preempt_pending(); } @@ -554,9 +542,7 @@ impl TaskInner { fn current_check_preempt_pending() { use ax_kernel_guard::NoPreemptIrqSave; let curr = crate::current(); - if (curr.force_resched_pending() || curr.need_resched.load(Ordering::Acquire)) - && curr.can_preempt(0) - { + if (curr.force_resched_pending() || curr.core.preempt_pending()) && curr.can_preempt(0) { // Note: if we want to print log msg during `preempt_resched`, we have to // disable preemption here, because the ax-log may cause preemption. let mut rq = crate::current_run_queue::(); @@ -564,7 +550,7 @@ impl TaskInner { #[cfg(all(feature = "smp", feature = "ipi"))] crate::run_queue::clear_remote_reschedule_pending_for_current_cpu(); rq.force_resched() - } else if curr.need_resched.load(Ordering::Acquire) { + } else if curr.core.preempt_pending() { rq.preempt_resched() } } @@ -572,6 +558,7 @@ impl TaskInner { /// Notify all tasks that join on this task. pub(crate) fn notify_exit(&self, exit_code: i32) { + self.expire_irq_wakers(); self.set_state(TaskState::Exited); self.exit_code.store(exit_code, Ordering::Release); self.wait_for_exit.notify_all(false); @@ -602,7 +589,7 @@ impl TaskInner { #[cfg(feature = "smp")] #[inline] pub(crate) fn set_cpu_id(&self, cpu_id: u32) { - self.cpu_id.store(cpu_id, Ordering::Release); + self.core.set_cpu_id(CpuId(cpu_id as usize)); } /// Returns whether the task is running on a CPU. @@ -614,33 +601,70 @@ impl TaskInner { #[cfg(feature = "smp")] #[inline] pub(crate) fn on_cpu(&self) -> bool { - self.on_cpu.load(Ordering::Acquire) + self.core.on_cpu() } /// Sets whether the task is running on a CPU. #[cfg(feature = "smp")] #[inline] pub(crate) fn set_on_cpu(&self, on_cpu: bool) { - self.on_cpu.store(on_cpu, Ordering::Release) + self.core.set_on_cpu(on_cpu) } } impl fmt::Debug for TaskInner { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("TaskInner") - .field("id", &self.id) + .field("id", &self.id()) .field("name", &self.name) .field("state", &self.state()) .finish() } } +fn ax_to_bare_cpumask(mask: AxCpuMask) -> CpuMask { + let mut bits = 0u64; + for cpu_id in 0..crate::build_info::CPU_CAPACITY.min(64) { + if mask.get(cpu_id) { + bits |= 1u64 << cpu_id; + } + } + CpuMask::from_bits(bits) +} + +fn bare_to_ax_cpumask(mask: CpuMask) -> AxCpuMask { + let mut ax_mask = AxCpuMask::new(); + for cpu_id in 0..crate::build_info::CPU_CAPACITY.min(64) { + ax_mask.set(cpu_id, mask.contains(CpuId(cpu_id))); + } + ax_mask +} + impl Drop for TaskInner { fn drop(&mut self) { + crate::irq_wake::expire_task_irq_wakers(self); debug!("task drop: {}", self.id_name()); } } +impl bare_task::TaskOps for TaskInner { + fn core(&self) -> &TaskCore { + &self.core + } + + fn id_name(&self) -> String { + self.id_name() + } + + fn is_idle(&self) -> bool { + self.is_idle + } + + fn is_init(&self) -> bool { + self.is_init + } +} + pub(crate) struct TaskStack { ptr: usize, size: usize, @@ -1008,8 +1032,8 @@ extern "C" fn task_entry() -> ! { // Clear the prev task on CPU before running the task entry function. crate::run_queue::clear_prev_task_on_cpu(); } - // Enable irq (if feature "irq" is enabled) before running the task entry function. - #[cfg(all(feature = "irq", not(feature = "host-test")))] + // Enable IRQs before running the task entry function. + #[cfg(not(feature = "host-test"))] ax_hal::asm::enable_irqs(); let task = crate::current(); if let Some(entry) = task.entry.take() { diff --git a/os/arceos/modules/axtask/src/tests.rs b/os/arceos/modules/axtask/src/tests.rs index f7be771281..7a605ee28c 100644 --- a/os/arceos/modules/axtask/src/tests.rs +++ b/os/arceos/modules/axtask/src/tests.rs @@ -1,10 +1,12 @@ -use core::sync::atomic::{AtomicUsize, Ordering}; -#[cfg(feature = "irq")] -use std::sync::{Arc, Barrier}; +use core::{ + future::poll_fn, + sync::atomic::{AtomicUsize, Ordering}, + task::{Context, Poll, Waker}, +}; use std::{ panic::{AssertUnwindSafe, catch_unwind, resume_unwind}, - sync::{OnceLock, mpsc}, - task::Context, + sync::{Arc, Barrier, OnceLock, mpsc}, + task::Wake, thread, }; @@ -13,9 +15,37 @@ use ax_errno::{AxError, AxResult}; use ax_kernel_guard::NoPreempt; use axpoll::{IoEvents, Pollable}; -#[cfg(feature = "irq")] -use crate::IrqNotify; -use crate::{WaitQueue, api as ax_task, current}; +use crate::{HardIrqSignal, WaitQueue, api as ax_task, current}; + +struct WakeCounter { + count: AtomicUsize, +} + +impl WakeCounter { + fn new() -> Arc { + Arc::new(Self { + count: AtomicUsize::new(0), + }) + } + + fn count(&self) -> usize { + self.count.load(Ordering::Acquire) + } + + fn waker(self: &Arc) -> Waker { + Waker::from(self.clone()) + } +} + +impl Wake for WakeCounter { + fn wake(self: Arc) { + self.count.fetch_add(1, Ordering::AcqRel); + } + + fn wake_by_ref(self: &Arc) { + self.count.fetch_add(1, Ordering::AcqRel); + } +} type TestResult = Result<(), Box>; type TestJob = (Box, mpsc::Sender); @@ -97,8 +127,8 @@ fn panic_payload_message(payload: &(dyn core::any::Any + Send)) -> &str { } #[test] -#[cfg(not(any(feature = "irq", feature = "preempt")))] -fn might_sleep_ignores_irq_state_without_irq_feature() { +#[cfg(not(feature = "preempt"))] +fn might_sleep_ignores_irq_state_without_preempt_feature() { run_in_test_scheduler(|| { assert_eq!(ax_task::in_atomic_context(), false); ax_task::might_sleep(); @@ -206,6 +236,312 @@ fn poll_io_nonblocking_wouldblock_wins_over_pending_interrupt() { }); } +#[test] +fn runtime_event_publish_before_poll_is_observed() { + let event = crate::local::RuntimeEvent::new(); + let seq = event.publish(0x1); + let counter = WakeCounter::new(); + let waker = counter.waker(); + let mut cx = Context::from_waker(&waker); + let observed = crate::local::RuntimeEventSeq::new(); + + assert_eq!(seq.get(), 1); + assert_eq!(event.poll_changed(&observed, &mut cx), Poll::Ready(1)); + assert_eq!(observed.get(), 1); + assert_eq!(event.take_bits(), 0x1); + assert_eq!(counter.count(), 0); +} + +#[test] +fn runtime_event_publish_after_register_wakes_waiter() { + let event = crate::local::RuntimeEvent::new(); + let observed = crate::local::RuntimeEventSeq::new(); + observed.update(event.seq()); + let counter = WakeCounter::new(); + let waker = counter.waker(); + let mut cx = Context::from_waker(&waker); + + assert_eq!(event.poll_changed(&observed, &mut cx), Poll::Pending); + event.publish(0x2); + + assert_eq!(counter.count(), 1); + assert_eq!(event.poll_changed(&observed, &mut cx), Poll::Ready(1)); + assert_eq!(event.take_bits(), 0x2); +} + +#[test] +fn runtime_event_wait_changed_observes_publish_before_wait() { + run_in_test_scheduler(|| { + let event = crate::local::RuntimeEvent::new(); + let observed = crate::local::RuntimeEventSeq::new(); + + event.publish_from_irq(0x4); + let seq = event.wait_changed(&observed); + + assert_eq!(seq.get(), 1); + assert_eq!(observed.get(), 1); + assert_eq!(event.take_bits(), 0x4); + }); +} + +#[test] +fn local_executor_defers_irq_wakers_until_task_context() { + let event = Arc::new(crate::local::RuntimeEvent::new()); + let executor = crate::local::LocalExecutor::new(); + let spawner = executor.spawner(); + let ran = Arc::new(AtomicUsize::new(0)); + + let task_event = event.clone(); + let task_ran = ran.clone(); + spawner + .spawn_local(async move { + let observed = crate::local::RuntimeEventSeq::new(); + poll_fn(|cx| task_event.poll_changed(&observed, cx)).await; + task_ran.fetch_add(1, Ordering::AcqRel); + }) + .expect("local spawn should fit in executor queue"); + + executor.run_until_idle(); + assert_eq!(ran.load(Ordering::Acquire), 0); + + event.publish_from_irq(0x4); + executor.run_until_idle(); + assert_eq!(ran.load(Ordering::Acquire), 0); + + event.wake_waiters_deferred(); + executor.run_until_idle(); + assert_eq!(ran.load(Ordering::Acquire), 1); +} + +#[test] +fn local_executor_repolls_after_irq_runtime_event() { + let event = Arc::new(crate::local::RuntimeEvent::new()); + let executor = crate::local::LocalExecutor::new(); + let spawner = executor.spawner(); + let ran = Arc::new(AtomicUsize::new(0)); + + let task_event = event.clone(); + let task_ran = ran.clone(); + spawner + .spawn_local(async move { + let observed = crate::local::RuntimeEventSeq::new(); + poll_fn(|cx| task_event.poll_changed(&observed, cx)).await; + task_ran.fetch_add(1, Ordering::AcqRel); + }) + .expect("local spawn should fit in executor queue"); + + executor.run_until_idle(); + assert_eq!(ran.load(Ordering::Acquire), 0); + + event.publish_from_irq(0x8); + executor.run_until_event(&event, || event.has_unseen_events()); + + assert_eq!(ran.load(Ordering::Acquire), 1); + assert_eq!(event.take_bits(), 0x8); +} + +#[test] +fn local_executor_new_outside_task_context_does_not_panic() { + let executor = crate::local::LocalExecutor::new(); + executor.run_until_idle(); +} + +#[test] +fn runtime_event_publish_from_irq_with_wakes_host_task() { + run_in_test_scheduler(|| { + let event = crate::local::RuntimeEvent::new(); + let task_waker = crate::current_task_waker(); + let waker = task_waker.to_hard_irq_waker(); + + let (seq, wake) = event.publish_from_irq_with(0x20, &waker); + + assert_eq!(seq.get(), 1); + assert!(wake.woke()); + assert_eq!(event.take_bits(), 0x20); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 0); + }); +} + +#[test] +fn irq_task_waker_coalesces_and_preserves_bits() { + run_in_test_scheduler(|| { + let waker = crate::current_task_waker().to_hard_irq_waker(); + let initial_seq = waker.seq(); + + let first = waker.wake_from_irq(0x1); + let second = waker.wake_from_irq(0x2); + + assert!(first.woke()); + assert!(!second.woke()); + assert!(first.should_resched()); + assert_eq!(waker.seq(), initial_seq + 2); + assert_eq!(waker.take_bits(), 0x3); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 0); + }); +} + +#[test] +fn irq_task_waker_restores_current_task_blocked_before_resched() { + run_in_test_scheduler(|| { + let current = current().clone(); + let waker = crate::current_task_waker().to_hard_irq_waker(); + + current.set_state(crate::TaskState::Blocked); + let result = waker.wake_from_irq(0x4); + + assert!(result.woke()); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 1); + assert_eq!(current.state(), crate::TaskState::Running); + assert_eq!(waker.take_bits(), 0x4); + }); +} + +#[test] +fn irq_task_waker_does_not_keep_task_alive() { + let task = crate::TaskInner::new(|| {}, "irq-wake-weak-test".into(), RAW_TASK_STACK_SIZE); + let task = task.into_arc(); + assert_eq!(Arc::strong_count(&task), 1); + + let waker = crate::wake::TaskWaker::new(task.clone()).to_hard_irq_waker(); + assert_eq!( + Arc::strong_count(&task), + 1, + "IRQ wakers must not keep exited tasks alive", + ); + drop(task); + + assert!(!waker.wake_from_irq(0x1).woke()); + assert_eq!(waker.seq(), 0); + assert_eq!(waker.take_bits(), 0); +} + +#[test] +fn irq_task_waker_wakes_blocked_future_task() { + run_in_test_scheduler(|| { + let task = crate::TaskInner::new(|| {}, "irq-wake-test".into(), RAW_TASK_STACK_SIZE); + let task = task.into_arc(); + crate::register_task(&task); + task.set_state(crate::TaskState::Blocked); + let waker = crate::wake::TaskWaker::new(task.clone()).to_hard_irq_waker(); + + let result = waker.wake_from_irq(0x10); + assert!(result.should_resched()); + assert_eq!(waker.take_bits(), 0x10); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 1); + assert_eq!(task.state(), crate::TaskState::Ready); + }); +} + +#[test] +fn task_timer_wakeup_is_deferred_through_irq_wake_queue() { + run_in_test_scheduler(|| { + let task = current().clone(); + task.set_state(crate::TaskState::Blocked); + + crate::timers::set_alarm_wakeup(ax_hal::time::monotonic_time(), task.clone()); + crate::timers::check_events(false); + + assert_eq!( + task.state(), + crate::TaskState::Blocked, + "timer hard IRQ must only enqueue an IRQ wake before epilogue drain", + ); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 1); + assert_eq!(task.state(), crate::TaskState::Running); + }); +} + +#[test] +fn irq_task_waker_task_context_wake_bypasses_irq_queue() { + run_in_test_scheduler(|| { + let task = + crate::TaskInner::new(|| {}, "task-context-wake-test".into(), RAW_TASK_STACK_SIZE); + let task = task.into_arc(); + crate::register_task(&task); + task.set_state(crate::TaskState::Blocked); + let waker = crate::wake::TaskWaker::new(task.clone()); + + let result = waker.wake(0x80); + assert!(result.woke()); + assert_eq!(waker.take_bits(), 0x80); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 0); + assert_eq!(task.state(), crate::TaskState::Ready); + }); +} + +#[cfg(all(feature = "smp", feature = "host-test"))] +#[test] +fn irq_task_waker_respects_affinity_changed_while_blocked() { + let _remote_wake_guard = crate::run_queue::remote_wake_test_guard(); + run_in_test_scheduler(|| { + const REMOTE_CPU: usize = 1; + + crate::run_queue::init_test_run_queue_for_cpu(REMOTE_CPU); + + let task = + crate::TaskInner::new(|| {}, "irq-wake-affinity-test".into(), RAW_TASK_STACK_SIZE); + let task = task.into_arc(); + crate::register_task(&task); + task.set_state(crate::TaskState::Blocked); + task.set_cpumask(crate::AxCpuMask::one_shot(REMOTE_CPU)); + let waker = crate::wake::TaskWaker::new(task.clone()).to_hard_irq_waker(); + + let result = waker.wake_from_irq(0x20); + assert!(result.should_resched()); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 1); + + assert_eq!(task.state(), crate::TaskState::Ready); + assert_eq!(task.cpu_id() as usize, REMOTE_CPU); + }); +} + +#[test] +fn irq_task_waker_is_invalid_after_logical_exit() { + run_in_test_scheduler(|| { + let task = crate::TaskInner::new(|| {}, "irq-wake-exit-test".into(), RAW_TASK_STACK_SIZE); + let task = task.into_arc(); + crate::register_task(&task); + let waker = crate::wake::TaskWaker::new(task.clone()).to_hard_irq_waker(); + + task.notify_exit(0); + + let result = waker.wake_from_irq(0x40); + assert!(!result.woke()); + assert_eq!(waker.seq(), 0); + assert_eq!(waker.take_bits(), 0); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 0); + }); +} + +#[test] +fn test_irq_notify_rebinds_after_first_waiter_exits() { + run_in_test_scheduler(|| { + let notify = HardIrqSignal::new(); + let first = + crate::TaskInner::new(|| {}, "irq-notify-first-waiter".into(), RAW_TASK_STACK_SIZE); + let first = first.into_arc(); + crate::register_task(&first); + let first_waker = crate::wake::TaskWaker::new(first.clone()).to_hard_irq_waker(); + notify.arm_irq_waker_for_test(first_waker); + first.notify_exit(0); + + let second = crate::TaskInner::new( + || {}, + "irq-notify-second-waiter".into(), + RAW_TASK_STACK_SIZE, + ); + let second = second.into_arc(); + crate::register_task(&second); + second.set_state(crate::TaskState::Blocked); + let second_waker = crate::wake::TaskWaker::new(second.clone()).to_hard_irq_waker(); + notify.arm_irq_waker_for_test(second_waker); + + notify.notify_irq(); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 1); + assert_eq!(second.state(), crate::TaskState::Ready); + }); +} + #[test] fn test_sched_fifo() { run_in_test_scheduler(|| { @@ -308,13 +644,12 @@ fn test_wait_queue() { }); } -#[cfg(feature = "irq")] #[test] fn test_irq_notify_coalesces_concurrent_irq_callbacks() { const NUM_IRQ_THREADS: usize = 8; const NOTIFIES_PER_THREAD: usize = 32; - let notify = Arc::new(IrqNotify::new()); + let notify = Arc::new(HardIrqSignal::new()); let barrier = Arc::new(Barrier::new(NUM_IRQ_THREADS)); let mut handles = Vec::with_capacity(NUM_IRQ_THREADS); @@ -338,11 +673,10 @@ fn test_irq_notify_coalesces_concurrent_irq_callbacks() { assert!(!notify.drain()); } -#[cfg(feature = "irq")] #[test] fn test_irq_notify_wait_observes_notify_before_wait() { run_in_test_scheduler(|| { - let notify = IrqNotify::new(); + let notify = HardIrqSignal::new(); notify.notify_irq(); notify.wait(); @@ -352,74 +686,188 @@ fn test_irq_notify_wait_observes_notify_before_wait() { }); } -#[cfg(feature = "irq")] #[test] fn test_irq_notify_wakes_sleeping_deferred_worker() { run_in_test_scheduler(|| { - let notify = Arc::new(IrqNotify::new()); - let started_wq = Arc::new(WaitQueue::new()); - let started = Arc::new(AtomicUsize::new(0)); - let finished = Arc::new(AtomicUsize::new(0)); + let notify = HardIrqSignal::new(); + let task = crate::TaskInner::new( + || {}, + "irq-notify-sleeping-worker".into(), + RAW_TASK_STACK_SIZE, + ); + let task = task.into_arc(); + crate::register_task(&task); + task.set_state(crate::TaskState::Blocked); + notify + .arm_irq_waker_for_test(crate::wake::TaskWaker::new(task.clone()).to_hard_irq_waker()); - let worker = { - let notify = notify.clone(); - let started_wq = started_wq.clone(); - let started = started.clone(); - let finished = finished.clone(); - ax_task::spawn(move || { - started.store(1, Ordering::Release); - started_wq.notify_one(true); + notify.notify_irq(); - notify.wait(); + assert!(notify.is_pending()); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 1); + assert_eq!(task.state(), crate::TaskState::Ready); + assert!(notify.drain()); + }); +} - finished.store(1, Ordering::Release); - }) - }; +#[test] +fn test_future_blocked_resched_aborts_when_event_arrives_before_sleep() { + run_in_test_scheduler(|| { + let checks = AtomicUsize::new(0); - started_wq.wait_until(|| started.load(Ordering::Acquire) == 1); - assert_eq!(finished.load(Ordering::Acquire), 0); + crate::current_run_queue::() + .future_blocked_resched(|| checks.fetch_add(1, Ordering::AcqRel) == 0); - notify.notify_irq(); - for _ in 0..64 { - if finished.load(Ordering::Acquire) == 1 { - break; - } - ax_task::yield_now(); - } + assert_eq!(checks.load(Ordering::Acquire), 1); + assert_eq!(current().state(), crate::TaskState::Running); + assert!(!current().in_wait_queue()); + }); +} - assert_eq!(finished.load(Ordering::Acquire), 1); - assert!(!notify.drain()); - assert_eq!(worker.join(), 0); +#[test] +fn test_wait_until_rechecks_after_queueing_without_sleeping() { + run_in_test_scheduler(|| { + let wait_queue = WaitQueue::new(); + let checks = AtomicUsize::new(0); + + wait_queue.wait_until(|| checks.fetch_add(1, Ordering::AcqRel) != 0); + + assert!(checks.load(Ordering::Acquire) >= 2); + assert_eq!(current().state(), crate::TaskState::Running); + assert!(!current().in_wait_queue()); + assert!(!wait_queue.notify_one(false)); }); } -#[cfg(feature = "irq")] #[test] -fn test_irq_notify_wakes_after_concurrent_irq_callbacks() { +fn test_blocked_resched_abortable_removes_waiter_before_return() { run_in_test_scheduler(|| { - const NUM_IRQ_THREADS: usize = 6; + let wait_queue = ax_kspin::SpinNoIrq::new(bare_task::WaitQueueCore::new()); + let aborted = crate::current_run_queue::() + .blocked_resched_abortable(&wait_queue, || true); + + assert!(aborted); + assert_eq!(current().state(), crate::TaskState::Running); + assert!(!current().in_wait_queue()); + assert!(wait_queue.lock().is_empty()); + }); +} - let notify = Arc::new(IrqNotify::new()); - let started_wq = Arc::new(WaitQueue::new()); - let started = Arc::new(AtomicUsize::new(0)); - let finished = Arc::new(AtomicUsize::new(0)); +#[test] +fn test_stale_wait_queue_entry_does_not_clear_new_wait_membership() { + run_in_test_scheduler(|| { + let stale_queue = WaitQueue::new(); + let active_queue = WaitQueue::new(); + let task = + crate::TaskInner::new(|| {}, "stale-key-waiter".into(), RAW_TASK_STACK_SIZE).into_arc(); + + crate::register_task(&task); + task.set_state(crate::TaskState::Blocked); + stale_queue.push_task_for_test(task.clone()); + active_queue.push_task_for_test(task.clone()); + + assert!(task.in_wait_queue()); + assert!(!stale_queue.notify_one(false)); + assert!(task.in_wait_queue()); + assert!(active_queue.notify_one(false)); + assert_eq!(task.state(), crate::TaskState::Ready); + assert!(!task.in_wait_queue()); + }); +} - let worker = { - let notify = notify.clone(); - let started_wq = started_wq.clone(); - let started = started.clone(); - let finished = finished.clone(); - ax_task::spawn(move || { - started.store(1, Ordering::Release); - started_wq.notify_one(true); +#[test] +fn test_wake_task_clears_raw_wait_queue_membership() { + run_in_test_scheduler(|| { + let wait_queue = WaitQueue::new(); + let task = crate::TaskInner::new( + || {}, + "raw-waitqueue-interrupt-test".into(), + RAW_TASK_STACK_SIZE, + ); + let task = task.into_arc(); + crate::register_task(&task); + task.set_state(crate::TaskState::Blocked); + wait_queue.push_task_for_test(task.clone()); - notify.wait(); + assert!(task.in_wait_queue()); + crate::wake_task(&task); - finished.fetch_add(1, Ordering::Release); - }) - }; + assert_eq!(task.state(), crate::TaskState::Ready); + assert!(!task.in_wait_queue()); + assert!(!wait_queue.notify_one(false)); + }); +} - started_wq.wait_until(|| started.load(Ordering::Acquire) == 1); +#[test] +fn test_wait_queue_notify_one_ignores_ready_stale_waiter() { + run_in_test_scheduler(|| { + let wait_queue = WaitQueue::new(); + let stale = crate::TaskInner::new(|| {}, "ready-stale-waiter".into(), RAW_TASK_STACK_SIZE) + .into_arc(); + let blocked = + crate::TaskInner::new(|| {}, "blocked-waiter".into(), RAW_TASK_STACK_SIZE).into_arc(); + + crate::register_task(&stale); + crate::register_task(&blocked); + stale.set_state(crate::TaskState::Ready); + blocked.set_state(crate::TaskState::Blocked); + wait_queue.push_task_for_test(stale.clone()); + wait_queue.push_task_for_test(blocked.clone()); + + assert!(wait_queue.notify_one(false)); + assert_eq!(blocked.state(), crate::TaskState::Ready); + assert!(!stale.in_wait_queue()); + assert!(!blocked.in_wait_queue()); + assert!(!wait_queue.notify_one(false)); + }); +} + +#[test] +fn test_wait_queue_notify_one_with_ignores_ready_stale_waiter() { + run_in_test_scheduler(|| { + let wait_queue = WaitQueue::new(); + let stale = + crate::TaskInner::new(|| {}, "ready-stale-with".into(), RAW_TASK_STACK_SIZE).into_arc(); + let blocked = + crate::TaskInner::new(|| {}, "blocked-with".into(), RAW_TASK_STACK_SIZE).into_arc(); + + crate::register_task(&stale); + crate::register_task(&blocked); + stale.set_state(crate::TaskState::Ready); + blocked.set_state(crate::TaskState::Blocked); + wait_queue.push_task_for_test(stale.clone()); + wait_queue.push_task_for_test(blocked.clone()); + + let observed = AtomicUsize::new(usize::MAX); + assert!(wait_queue.notify_one_with(false, |task_id| { + observed.store(task_id as usize, Ordering::Release); + })); + assert_eq!( + observed.load(Ordering::Acquire), + blocked.id().as_u64() as usize + ); + assert_eq!(blocked.state(), crate::TaskState::Ready); + assert!(!stale.in_wait_queue()); + assert!(!blocked.in_wait_queue()); + }); +} + +#[test] +fn test_irq_notify_wakes_after_concurrent_irq_callbacks() { + run_in_test_scheduler(|| { + const NUM_IRQ_THREADS: usize = 6; + + let notify = Arc::new(HardIrqSignal::new()); + let task = crate::TaskInner::new( + || {}, + "irq-notify-concurrent-waiter".into(), + RAW_TASK_STACK_SIZE, + ); + let task = task.into_arc(); + crate::register_task(&task); + task.set_state(crate::TaskState::Blocked); + notify + .arm_irq_waker_for_test(crate::wake::TaskWaker::new(task.clone()).to_hard_irq_waker()); let barrier = Arc::new(Barrier::new(NUM_IRQ_THREADS)); let mut handles = Vec::with_capacity(NUM_IRQ_THREADS); @@ -435,21 +883,14 @@ fn test_irq_notify_wakes_after_concurrent_irq_callbacks() { handle.join().unwrap(); } - for _ in 0..64 { - if finished.load(Ordering::Acquire) == 1 { - break; - } - ax_task::yield_now(); - } - - assert_eq!(finished.load(Ordering::Acquire), 1); - assert_eq!(worker.join(), 0); + assert!(notify.is_pending()); + assert_eq!(crate::drain_irq_wake_queue_current_cpu(), 1); + assert_eq!(task.state(), crate::TaskState::Ready); }); } -#[cfg(feature = "irq")] #[test] -fn test_wait_queue_irq_notify_all_wakes_sleepers() { +fn test_wait_queue_deferred_notify_all_wakes_sleepers() { run_in_test_scheduler(|| { const NUM_SLEEPERS: usize = 4; @@ -480,7 +921,7 @@ fn test_wait_queue_irq_notify_all_wakes_sleepers() { assert_eq!(finished.load(Ordering::Acquire), 0); released.store(true, Ordering::Release); - wait_queue.notify_all_from_irq(); + wait_queue.notify_all_deferred(); for sleeper in sleepers { assert_eq!(sleeper.join(), 0); } diff --git a/os/arceos/modules/axtask/src/timers.rs b/os/arceos/modules/axtask/src/timers.rs index 961a4e6a36..2ae3b1fd99 100644 --- a/os/arceos/modules/axtask/src/timers.rs +++ b/os/arceos/modules/axtask/src/timers.rs @@ -2,12 +2,10 @@ use alloc::{boxed::Box, vec::Vec}; use core::sync::atomic::{AtomicU64, Ordering}; use ax_hal::time::{TimeValue, monotonic_time}; -use ax_kernel_guard::{NoOp, NoPreemptIrqSave}; +use ax_kernel_guard::NoPreemptIrqSave; use ax_timer_list::{TimerEvent, TimerList}; -#[cfg(feature = "smp")] -use crate::select_run_queue; -use crate::{AxTaskRef, current_run_queue}; +use crate::{AxTaskRef, HardIrqWaker, TaskWaker}; static TIMER_TICKET_ID: AtomicU64 = AtomicU64::new(1); @@ -20,6 +18,7 @@ percpu_static! { struct TaskWakeupEvent { ticket_id: u64, task: AxTaskRef, + irq_waker: HardIrqWaker, } impl TimerEvent for TaskWakeupEvent { @@ -33,28 +32,12 @@ impl TimerEvent for TaskWakeupEvent { return; } - // Timer ticket match. Timers are per-CPU, so prefer waking the task on - // the CPU that owns and expires this timer event. Falling back to the - // affinity selector is only needed if the task's affinity changed while - // it was sleeping. - wake_task_from_timer(self.task) + // Timer interrupts are hard IRQ context. Do not touch scheduler or + // wait-queue locks here; the IRQ epilogue drains the wake queue. + let _ = self.irq_waker.wake_from_irq(0); } } -#[cfg(feature = "smp")] -fn wake_task_from_timer(task: AxTaskRef) { - if task.cpumask().get(ax_hal::percpu::this_cpu_id()) { - current_run_queue::().unblock_task(task, true); - } else { - select_run_queue::(&task).unblock_task(task, true); - } -} - -#[cfg(not(feature = "smp"))] -fn wake_task_from_timer(task: AxTaskRef) { - current_run_queue::().unblock_task(task, true); -} - /// Registers a callback function to be called on each timer tick. pub fn register_timer_callback(callback: F) where @@ -105,10 +88,18 @@ pub(crate) fn next_deadline_nanos() -> Option { pub(crate) fn set_alarm_wakeup(deadline: TimeValue, task: AxTaskRef) { let _g = NoPreemptIrqSave::new(); + let irq_waker = TaskWaker::new(task.clone()).to_hard_irq_waker(); TIMER_LIST.with_current(|timer_list| { let ticket_id = TIMER_TICKET_ID.fetch_add(1, Ordering::AcqRel); task.set_timer_ticket(ticket_id); - timer_list.set(deadline, TaskWakeupEvent { ticket_id, task }); + timer_list.set( + deadline, + TaskWakeupEvent { + ticket_id, + task, + irq_waker, + }, + ); }); maybe_reprogram_timer(deadline); } diff --git a/os/arceos/modules/axtask/src/wait_queue.rs b/os/arceos/modules/axtask/src/wait_queue.rs index 21f1de3a1f..4e91eac561 100644 --- a/os/arceos/modules/axtask/src/wait_queue.rs +++ b/os/arceos/modules/axtask/src/wait_queue.rs @@ -1,7 +1,6 @@ -use alloc::collections::VecDeque; - use ax_kernel_guard::NoPreemptIrqSave; use ax_kspin::{SpinNoIrq, SpinNoIrqGuard}; +use bare_task::WaitQueueCore; use crate::{AxTaskRef, CurrentTask, current_run_queue, select_wake_run_queue}; @@ -29,10 +28,10 @@ use crate::{AxTaskRef, CurrentTask, current_run_queue, select_wake_run_queue}; /// assert_eq!(VALUE.load(Ordering::Acquire), 1); /// ``` pub struct WaitQueue { - queue: SpinNoIrq>, + queue: SpinNoIrq>, } -pub(crate) type WaitQueueGuard<'a> = SpinNoIrqGuard<'a, VecDeque>; +pub(crate) type WaitQueueGuard<'a> = SpinNoIrqGuard<'a, WaitQueueCore>; impl Default for WaitQueue { fn default() -> Self { @@ -44,23 +43,27 @@ impl WaitQueue { /// Creates an empty wait queue. pub const fn new() -> Self { Self { - queue: SpinNoIrq::new(VecDeque::new()), + queue: SpinNoIrq::new(WaitQueueCore::new()), } } + fn key(&self) -> usize { + core::ptr::addr_of!(self.queue) as usize + } + /// Cancel events by removing the task from the wait queue. /// If `from_timer_list` is true, try to remove the task from the timer list. fn cancel_events(&self, curr: CurrentTask, _from_timer_list: bool) { // A task can be woken by only one event (timer or `notify()`), so remove it from the other queue. - if curr.in_wait_queue() { + let wait_queue_key = self.key(); + if curr.is_waiting_on(wait_queue_key) { // wake up by timer (timeout). self.queue.lock().retain(|t| !curr.ptr_eq(t)); - curr.set_in_wait_queue(false); + curr.clear_wait_queue_key(wait_queue_key); } // Try to cancel a timer event from timer lists. // Just mark task's current timer ticket ID as expired. - #[cfg(feature = "irq")] if _from_timer_list { curr.timer_ticket_expired(); // Note: @@ -76,7 +79,7 @@ impl WaitQueue { #[track_caller] pub fn wait(&self) { crate::api::might_sleep(); - current_run_queue::().blocked_resched(self.queue.lock()); + current_run_queue::().blocked_resched(self.queue.lock(), self.key()); self.cancel_events(crate::current(), false); } @@ -93,13 +96,11 @@ impl WaitQueue { crate::api::might_sleep(); let curr = crate::current(); loop { - let mut rq = current_run_queue::(); - let wq = self.queue.lock(); if condition() { break; } - - rq.blocked_resched(wq); + let mut rq = current_run_queue::(); + rq.blocked_resched_abortable(&self.queue, &condition); // Preemption may occur here. } self.cancel_events(curr, false); @@ -107,7 +108,6 @@ impl WaitQueue { /// Blocks the current task and put it into the wait queue, until other tasks /// notify it, or the given duration has elapsed. - #[cfg(feature = "irq")] #[track_caller] pub fn wait_timeout(&self, dur: core::time::Duration) -> bool { crate::api::might_sleep(); @@ -121,7 +121,7 @@ impl WaitQueue { ); let timeout = loop { crate::timers::set_alarm_wakeup(deadline, curr.clone()); - rq.blocked_resched(self.queue.lock()); + rq.blocked_resched(self.queue.lock(), self.key()); // Still in the wait queue means the timer path woke us. Re-check // the monotonic deadline so an early wake cannot truncate sleeps. @@ -143,7 +143,6 @@ impl WaitQueue { /// /// Note that even other tasks notify this task, it will not wake up until /// the above conditions are met. - #[cfg(feature = "irq")] #[track_caller] pub fn wait_timeout_until(&self, dur: core::time::Duration, condition: F) -> bool where @@ -159,18 +158,19 @@ impl WaitQueue { ); let mut timeout = true; loop { - let mut rq = current_run_queue::(); if ax_hal::time::monotonic_time() >= deadline { break; } - let wq = self.queue.lock(); if condition() { timeout = false; break; } + let mut rq = current_run_queue::(); crate::timers::set_alarm_wakeup(deadline, curr.clone()); - rq.blocked_resched(wq); + rq.blocked_resched_abortable(&self.queue, || { + condition() || ax_hal::time::monotonic_time() >= deadline + }); // Preemption may occur here. } // Always try to remove the task from the timer list. @@ -182,31 +182,35 @@ impl WaitQueue { /// If `resched` is true, the current task will be preempted when the /// preemption is enabled. pub fn notify_one(&self, resched: bool) -> bool { - let task = self.pop_front(); - if let Some(task) = task { - unblock_one_task(task, resched); - return true; + while let Some(task) = self.pop_front() { + if unblock_one_task(task, resched) { + return true; + } } false } - /// Wakes up one task from IRQ context. + /// Wakes up one task from deferred IRQ or task context. /// - /// This method is intended for low-level deferred notification paths. It - /// only unblocks the worker and marks the current task for rescheduling - /// after IRQ/preemption guards are released; it must not be used as a - /// substitute for publishing the condition that the waiter will observe. - pub fn notify_one_from_irq(&self) -> bool { + /// This is **not** a hard-IRQ-safe wake path: it may take wait-queue, + /// run-queue, and scheduler locks. Hard IRQ handlers must publish device + /// state and use [`HardIrqWaker`](crate::HardIrqWaker) or + /// [`HardIrqSignal`](crate::HardIrqSignal) instead. + pub fn notify_one_deferred(&self) -> bool { + debug_assert!( + !ax_hal::irq::in_irq_context(), + "WaitQueue::notify_one_deferred is not hard-IRQ-context safe; use HardIrqWaker", + ); self.notify_one(true) } /// Wakes up one task in the wait queue and runs a callback on it. /// - /// The callback `func` is invoked while holding the wait-queue lock and - /// before the selected task is unblocked. It receives the task's ID as a - /// `u64` when a task is available, or `0` if the wait queue is empty. - /// This can be used for lock handoff or other bookkeeping associated with - /// the waking task. + /// The callback `func` receives the task ID only when a task is actually + /// unblocked. Stale entries that no longer belong to this queue, or tasks + /// that have already been made runnable by another wake path, are skipped + /// and do not consume this notification. The callback receives `0` only if + /// no task can be woken. /// /// If `resched` is true, the current task will be preempted when the /// preemption is enabled. @@ -214,25 +218,14 @@ impl WaitQueue { where F: Fn(u64), { - let task = { - let mut wq = self.queue.lock(); - match wq.pop_front() { - Some(task) => { - func(task.id().as_u64()); - task.set_in_wait_queue(false); - Some(task) - } - None => { - func(0); - None - } + while let Some(task) = self.pop_front() { + let task_id = task.id().as_u64(); + if unblock_one_task(task, resched) { + func(task_id); + return true; } - }; - - if let Some(task) = task { - unblock_one_task(task, resched); - return true; } + func(0); false } @@ -245,27 +238,41 @@ impl WaitQueue { } } - /// Wakes all tasks from IRQ context. + /// Wakes all tasks from deferred IRQ or task context. /// - /// This method is intended for low-level deferred notification paths. It - /// only unblocks workers and marks the current task for rescheduling after - /// IRQ/preemption guards are released; it must not be used as a substitute - /// for publishing the condition that waiters will observe. - pub fn notify_all_from_irq(&self) { - while self.notify_one_from_irq() { + /// This is **not** a hard-IRQ-safe wake path: it repeatedly calls + /// [`notify_one_deferred`](Self::notify_one_deferred) and therefore may take + /// scheduler locks. Hard IRQ handlers must publish state and wake an + /// IRQ-safe task waker instead. + pub fn notify_all_deferred(&self) { + debug_assert!( + !ax_hal::irq::in_irq_context(), + "WaitQueue::notify_all_deferred is not hard-IRQ-context safe; use HardIrqWaker", + ); + while self.notify_one_deferred() { // loop until the wait queue is empty } } fn pop_front(&self) -> Option { + let wait_queue_key = self.key(); let mut wq = self.queue.lock(); - let task = wq.pop_front()?; - task.set_in_wait_queue(false); - Some(task) + while let Some(task) = wq.pop_front() { + if task.clear_wait_queue_key(wait_queue_key) { + return Some(task); + } + } + None + } + + #[cfg(test)] + pub(crate) fn push_task_for_test(&self, task: AxTaskRef) { + task.set_wait_queue_key(self.key()); + self.queue.lock().push_back(task); } } -fn unblock_one_task(task: AxTaskRef, resched: bool) { +fn unblock_one_task(task: AxTaskRef, resched: bool) -> bool { // Select run queue by the CPU set of the task. select_wake_run_queue::(&task).unblock_task(task, resched) } diff --git a/os/arceos/ulib/axlibc/Cargo.toml b/os/arceos/ulib/axlibc/Cargo.toml index 08d0070df3..0e39d6f4f2 100644 --- a/os/arceos/ulib/axlibc/Cargo.toml +++ b/os/arceos/ulib/axlibc/Cargo.toml @@ -27,9 +27,6 @@ smp = ["ax-posix-api/smp"] # Floating point/SIMD fp-simd = ["ax-feat/fp-simd"] -# Interrupts -irq = ["ax-posix-api/irq", "ax-feat/irq"] - # Memory alloc = ["ax-posix-api/alloc"] tls = ["alloc", "ax-feat/tls"] diff --git a/os/arceos/ulib/axlibc/src/lib.rs b/os/arceos/ulib/axlibc/src/lib.rs index c1bf8be2fe..f54870ea58 100644 --- a/os/arceos/ulib/axlibc/src/lib.rs +++ b/os/arceos/ulib/axlibc/src/lib.rs @@ -5,8 +5,6 @@ //! - CPU //! - `smp`: Enable SMP (symmetric multiprocessing) support. //! - `fp-simd`: Enable floating point and SIMD support. -//! - Interrupts: -//! - `irq`: Enable interrupt handling support. //! - Memory //! - `alloc`: Enable dynamic memory allocation. //! - `tls`: Enable thread-local storage. diff --git a/os/arceos/ulib/axstd/Cargo.toml b/os/arceos/ulib/axstd/Cargo.toml index 42781f9283..9c2cee312a 100644 --- a/os/arceos/ulib/axstd/Cargo.toml +++ b/os/arceos/ulib/axstd/Cargo.toml @@ -37,7 +37,6 @@ uspace = ["ax-feat/uspace"] hv = ["ax-feat/hv"] # Interrupts -irq = ["ax-api/irq", "ax-feat/irq", "ax-posix-api/irq"] ipi = ["ax-api/ipi", "ax-feat/ipi"] wake-ipi = ["ax-feat/wake-ipi"] diff --git a/os/arceos/ulib/axstd/src/lib.rs b/os/arceos/ulib/axstd/src/lib.rs index e3fc92a4b3..3004f750cf 100644 --- a/os/arceos/ulib/axstd/src/lib.rs +++ b/os/arceos/ulib/axstd/src/lib.rs @@ -14,7 +14,7 @@ //! - `smp`: Enable SMP (symmetric multiprocessing) support. //! - `fp-simd`: Enable floating point and SIMD support. //! - Interrupts: -//! - `irq`: Enable interrupt handling support. +//! - `ipi`: Enable Inter-Processor Interrupts (IPIs). //! - Memory //! - `alloc`: Enable dynamic memory allocation. //! - `paging`: Enable page table manipulation. diff --git a/os/arceos/ulib/axstd/src/thread/mod.rs b/os/arceos/ulib/axstd/src/thread/mod.rs index 1291e006a0..0b6b3f421d 100644 --- a/os/arceos/ulib/axstd/src/thread/mod.rs +++ b/os/arceos/ulib/axstd/src/thread/mod.rs @@ -29,8 +29,7 @@ pub fn exit(exit_code: i32) -> ! { /// Current thread is going to sleep for the given duration. /// -/// If one of `multitask` or `irq` features is not enabled, it uses busy-wait -/// instead. +/// Without `multitask`, this uses the single-task busy-wait sleep path instead. #[track_caller] pub fn sleep(dur: core::time::Duration) { sleep_until(ax_api::time::ax_monotonic_time() + dur); @@ -39,8 +38,7 @@ pub fn sleep(dur: core::time::Duration) { /// Current thread is going to sleep, it will be woken up at the given deadline. /// The deadline is measured against the monotonic clock. /// -/// If one of `multitask` or `irq` features is not enabled, it uses busy-wait -/// instead. +/// Without `multitask`, this uses the single-task busy-wait sleep path instead. #[track_caller] pub fn sleep_until(deadline: ax_api::time::AxTimeValue) { api::ax_sleep_until(deadline); diff --git a/os/axvisor/Cargo.toml b/os/axvisor/Cargo.toml index e87cdadead..1744daaafa 100644 --- a/os/axvisor/Cargo.toml +++ b/os/axvisor/Cargo.toml @@ -66,10 +66,10 @@ ax-crate-interface = { workspace = true } log = "0.4" # System dependent modules provided by ArceOS. -ax-std = { workspace = true, features = ["ext-ld", "paging", "irq", "multitask", "task-ext", "smp", "hv"] } -ax-hal = { workspace = true, features = ["paging", "irq", "smp", "hv", "axvisor-linker"] } +ax-std = { workspace = true, features = ["ext-ld", "paging", "multitask", "task-ext", "smp", "hv"] } +ax-hal = { workspace = true, features = ["paging", "smp", "hv", "axvisor-linker"] } # System dependent modules provided by ArceOS-Hypervisor (bare-metal only) -# ax-runtime = { version = "=0.3.0-preview.1", features = ["alloc", "irq", "paging", "smp", "multitask"] } +# ax-runtime = { version = "=0.3.0-preview.1", features = ["alloc", "paging", "smp", "multitask"] } axvm = { workspace = true, default-features = false } axvmconfig = { workspace = true, default-features = false } # System independent crates provided by ArceOS diff --git a/platforms/ax-plat/Cargo.toml b/platforms/ax-plat/Cargo.toml index 65403e6301..dc2f328905 100644 --- a/platforms/ax-plat/Cargo.toml +++ b/platforms/ax-plat/Cargo.toml @@ -11,14 +11,13 @@ license = "Apache-2.0" [features] smp = ["ax-kspin/smp"] -irq = ["dep:rdif-intc"] [dependencies] ax-memory-addr = { workspace = true } bitflags = "2.6" ax-crate-interface = { workspace = true } irq-framework = { workspace = true } -rdif-intc = { workspace = true, optional = true } +rdif-intc = { workspace = true } const-str = "1.0" ax-plat-macros = { workspace = true } ax-kernel-guard = { workspace = true } diff --git a/platforms/ax-plat/src/console.rs b/platforms/ax-plat/src/console.rs index 789adc9bdb..9b038b54ae 100644 --- a/platforms/ax-plat/src/console.rs +++ b/platforms/ax-plat/src/console.rs @@ -59,16 +59,13 @@ pub trait ConsoleIf { /// Returns the IRQ number for the console input interrupt. /// /// Returns `None` if input interrupt is not supported. - #[cfg(feature = "irq")] fn irq_num() -> Option; /// Enables or disables device-side console input interrupts. - #[cfg(feature = "irq")] fn set_input_irq_enabled(enabled: bool); /// Handles a console input IRQ in interrupt context and returns the /// corresponding device events. - #[cfg(feature = "irq")] fn handle_irq() -> ConsoleIrqEvent; } diff --git a/platforms/ax-plat/src/lib.rs b/platforms/ax-plat/src/lib.rs index 4589ca94d9..d44e91ecec 100644 --- a/platforms/ax-plat/src/lib.rs +++ b/platforms/ax-plat/src/lib.rs @@ -9,7 +9,6 @@ extern crate ax_plat_macros; pub mod console; pub mod init; -#[cfg(feature = "irq")] pub mod irq; pub mod mem; pub mod percpu; diff --git a/platforms/ax-plat/src/time.rs b/platforms/ax-plat/src/time.rs index 41ad0ab02c..9703fb20de 100644 --- a/platforms/ax-plat/src/time.rs +++ b/platforms/ax-plat/src/time.rs @@ -36,14 +36,12 @@ pub trait TimeIf { fn epochoffset_nanos() -> u64; /// Returns the IRQ number for the timer interrupt. - #[cfg(feature = "irq")] fn irq_num() -> irq_framework::IrqId; /// Set a one-shot timer. /// /// A timer interrupt will be triggered at the specified monotonic time /// deadline (in nanoseconds). - #[cfg(feature = "irq")] fn set_oneshot_timer(deadline_ns: u64); } diff --git a/platforms/axplat-dyn/Cargo.toml b/platforms/axplat-dyn/Cargo.toml index e69c3f445e..e6860f2ff5 100644 --- a/platforms/axplat-dyn/Cargo.toml +++ b/platforms/axplat-dyn/Cargo.toml @@ -10,9 +10,8 @@ license.workspace = true repository.workspace = true [features] -default = ["smp", "irq" ] +default = ["smp"] smp = ["ax-plat/smp"] -irq = ["ax-plat/irq"] rtc = [] efi = ["somehal/efi"] fp-simd = ["ax-cpu/fp-simd"] diff --git a/platforms/axplat-dyn/src/console.rs b/platforms/axplat-dyn/src/console.rs index 89c223af5b..d1f389d28d 100644 --- a/platforms/axplat-dyn/src/console.rs +++ b/platforms/axplat-dyn/src/console.rs @@ -1,8 +1,6 @@ -#[cfg(feature = "irq")] -use ax_plat::console::ConsoleIrqEvent; -use ax_plat::console::{ConsoleDeviceIdError, ConsoleDeviceIdResult, ConsoleIf}; +use ax_plat::console::{ConsoleDeviceIdError, ConsoleDeviceIdResult, ConsoleIf, ConsoleIrqEvent}; -#[cfg(all(feature = "irq", target_arch = "x86_64"))] +#[cfg(target_arch = "x86_64")] fn console_irq(raw: usize) -> Option { if let Some(gsi) = raw.checked_sub(rdrive::probe::acpi::PCI_INTX_VECTOR_BASE) { ax_plat::irq::resolve_irq_source(ax_plat::irq::IrqSource::AcpiGsi(gsi as u32)).ok() @@ -11,7 +9,7 @@ fn console_irq(raw: usize) -> Option { } } -#[cfg(all(feature = "irq", not(target_arch = "x86_64")))] +#[cfg(not(target_arch = "x86_64"))] fn console_irq(raw: usize) -> Option { Some(ax_plat::irq::IrqNumber(raw).expect("console IRQ exceeds legacy IRQ width")) } @@ -66,17 +64,14 @@ impl ConsoleIf for ConsoleIfImpl { /// Returns the IRQ number for the console input interrupt. /// /// Returns `None` if input interrupt is not supported. - #[cfg(feature = "irq")] fn irq_num() -> Option { somehal::console::irq_num().and_then(console_irq) } - #[cfg(feature = "irq")] fn set_input_irq_enabled(enabled: bool) { somehal::console::set_input_irq_enabled(enabled); } - #[cfg(feature = "irq")] fn handle_irq() -> ConsoleIrqEvent { let raw = somehal::console::handle_irq(); let mut event = ConsoleIrqEvent::empty(); @@ -93,7 +88,7 @@ impl ConsoleIf for ConsoleIfImpl { } } -#[cfg(all(test, feature = "irq", target_arch = "x86_64"))] +#[cfg(all(test, target_arch = "x86_64"))] mod tests { #[test] fn x86_console_irq_without_acpi_route_falls_back_to_polling() { diff --git a/platforms/axplat-dyn/src/generic_timer.rs b/platforms/axplat-dyn/src/generic_timer.rs index e357d41224..11ae3676ef 100644 --- a/platforms/axplat-dyn/src/generic_timer.rs +++ b/platforms/axplat-dyn/src/generic_timer.rs @@ -81,7 +81,6 @@ impl ax_plat::time::TimeIf for GenericTimer { } } /// Returns the IRQ number for the timer interrupt. - #[cfg(feature = "irq")] fn irq_num() -> ax_plat::irq::IrqId { somehal::irq::systick_irq() } @@ -89,7 +88,6 @@ impl ax_plat::time::TimeIf for GenericTimer { /// /// A timer interrupt will be triggered at the specified monotonic time /// deadline (in nanoseconds). - #[cfg(feature = "irq")] fn set_oneshot_timer(deadline_ns: u64) { let cnptct = somehal::timer::ticks() as u64; let deadline = GenericTimer::nanos_to_ticks(deadline_ns); diff --git a/platforms/axplat-dyn/src/lib.rs b/platforms/axplat-dyn/src/lib.rs index 98268fc875..75977ea8bf 100644 --- a/platforms/axplat-dyn/src/lib.rs +++ b/platforms/axplat-dyn/src/lib.rs @@ -15,7 +15,6 @@ mod console; pub mod drivers; mod generic_timer; mod init; -#[cfg(feature = "irq")] mod irq; mod mem; mod platform; @@ -24,13 +23,11 @@ mod power; pub use boot::{boot_stack_bounds, bootargs}; pub use generic_timer::try_init_epoch_offset; -#[cfg(feature = "irq")] pub fn enable_timer_irq() { somehal::timer::irq_enable(); } -#[cfg(feature = "irq")] pub fn ipi_irq() -> ax_plat::irq::IrqId { somehal::irq::ipi_irq() } -#[cfg(all(feature = "irq", target_arch = "riscv64", feature = "hv"))] +#[cfg(all(target_arch = "riscv64", feature = "hv"))] pub use irq::register_virtual_irq_injector; diff --git a/scripts/axbuild/src/axvisor/test/tests.rs b/scripts/axbuild/src/axvisor/test/tests.rs index 508e1b87e3..017efe8cfa 100644 --- a/scripts/axbuild/src/axvisor/test/tests.rs +++ b/scripts/axbuild/src/axvisor/test/tests.rs @@ -26,6 +26,13 @@ struct TestVmKernel { cmdline: String, } +#[derive(serde::Deserialize)] +struct TestQemuRuntimeConfig { + #[serde(default)] + success_regex: Vec, + shell_init_cmd: Option, +} + fn write_qemu_config(root: &Path, case: &str, arch: &str, body: &str) -> PathBuf { write_qemu_config_in_group(root, "normal", "default", case, arch, body) } @@ -673,6 +680,28 @@ fn x86_linux_direct_boot_configs_keep_timer_calibration_bypass() { } } +#[test] +fn x86_vmx_smoke_uses_prompt_success_without_delayed_shell_injection() { + let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .unwrap(); + let path = workspace_root.join("test-suit/axvisor/normal/qemu/smoke/qemu-x86_64-vmx.toml"); + let content = fs::read_to_string(&path).unwrap(); + let config: TestQemuRuntimeConfig = toml::from_str(&content).unwrap(); + let path = path.display(); + + assert!( + config.success_regex.iter().any(|pattern| pattern == "~ #"), + "{path} should treat reaching the Linux shell prompt as success" + ); + assert!( + config.shell_init_cmd.is_none(), + "{path} should not depend on delayed shell_init_cmd injection; nested VMX guests can \ + reboot after the first prompt before the delayed command is sent" + ); +} + #[test] fn ignores_qemu_only_build_groups_when_discovering_board_tests() { let root = tempdir().unwrap(); diff --git a/scripts/axbuild/src/clippy/mod.rs b/scripts/axbuild/src/clippy/mod.rs index fcfa68d8d4..6ba020a858 100644 --- a/scripts/axbuild/src/clippy/mod.rs +++ b/scripts/axbuild/src/clippy/mod.rs @@ -27,7 +27,7 @@ use timing::print_clippy_timing; pub(super) const DEFAULT_FEATURE: &str = "default"; pub(super) const AXSTD_STD_PACKAGE: &str = "ax-std"; pub(super) const AXSTD_STD_DEFAULT_FEATURE: &str = "default"; -pub(super) const AXSTD_STD_CLIPPY_FEATURES: &str = "std-compat,fs,multitask,irq,net"; +pub(super) const AXSTD_STD_CLIPPY_FEATURES: &str = "std-compat,fs,multitask,net"; pub(super) const AXSTD_STD_CLIPPY_TARGET: &str = "x86_64-unknown-none"; pub(super) const AX_HAL_PACKAGE: &str = "ax-hal"; diff --git a/scripts/axbuild/src/clippy/tests/expand.rs b/scripts/axbuild/src/clippy/tests/expand.rs index 2c311fb69f..e8f5ab41b1 100644 --- a/scripts/axbuild/src/clippy/tests/expand.rs +++ b/scripts/axbuild/src/clippy/tests/expand.rs @@ -467,11 +467,11 @@ fn docs_rs_targets_expand_base_and_feature_checks() { } #[test] -fn ax_hal_platform_features_are_filtered_by_target_arch() { +fn ax_hal_features_expand_for_each_docs_rs_target() { let checks = expand(&[pkg( "ax-hal", "ax-hal 0.1.0 (path+file:///tmp/ax-hal)", - &[("irq", &[])], + &[("fp-simd", &[])], Some(&["loongarch64-unknown-none", "riscv64gc-unknown-none-elf"]), )]); @@ -483,32 +483,35 @@ fn ax_hal_platform_features_are_filtered_by_target_arch() { }; assert!(has_feature_on_target( - "irq", + "fp-simd", "loongarch64-unknown-none-softfloat" )); - assert!(has_feature_on_target("irq", "riscv64gc-unknown-none-elf")); + assert!(has_feature_on_target( + "fp-simd", + "riscv64gc-unknown-none-elf" + )); } #[test] -fn ax_hal_target_only_features_are_skipped_for_host_clippy() { +fn ax_hal_features_expand_for_host_clippy() { let checks = expand(&[pkg( "ax-hal", "ax-hal 0.1.0 (path+file:///tmp/ax-hal)", - &[("irq", &[])], + &[("fp-simd", &[])], None, )]); assert!(checks.iter().any(|check| { - matches!(&check.kind, ClippyCheckKind::Feature(feature) if feature == "irq") + matches!(&check.kind, ClippyCheckKind::Feature(feature) if feature == "fp-simd") })); } #[test] -fn ax_hal_platform_feature_forwards_are_filtered_by_target_arch() { +fn ax_hal_feature_forwards_expand_for_each_docs_rs_target() { let checks = expand(&[pkg( "platform-forwarder", "platform-forwarder 0.1.0 (path+file:///tmp/platform-forwarder)", - &[("irq", &["ax-hal/irq"])], + &[("fp-simd", &["ax-hal/fp-simd"])], Some(&["loongarch64-unknown-none", "riscv64gc-unknown-none-elf"]), )]); @@ -520,10 +523,13 @@ fn ax_hal_platform_feature_forwards_are_filtered_by_target_arch() { }; assert!(has_feature_on_target( - "irq", + "fp-simd", "loongarch64-unknown-none-softfloat" )); - assert!(has_feature_on_target("irq", "riscv64gc-unknown-none-elf")); + assert!(has_feature_on_target( + "fp-simd", + "riscv64gc-unknown-none-elf" + )); } #[test] diff --git a/test-suit/arceos/c/build-aarch64-unknown-none-softfloat.toml b/test-suit/arceos/c/build-aarch64-unknown-none-softfloat.toml index 384d3c3fe7..d5a7c70c0d 100644 --- a/test-suit/arceos/c/build-aarch64-unknown-none-softfloat.toml +++ b/test-suit/arceos/c/build-aarch64-unknown-none-softfloat.toml @@ -3,7 +3,6 @@ features = [ "alloc", "paging", "multitask", - "irq", "fd", "pipe", "epoll", diff --git a/test-suit/arceos/c/build-loongarch64-unknown-none-softfloat.toml b/test-suit/arceos/c/build-loongarch64-unknown-none-softfloat.toml index a1c921671b..d5a7c70c0d 100644 --- a/test-suit/arceos/c/build-loongarch64-unknown-none-softfloat.toml +++ b/test-suit/arceos/c/build-loongarch64-unknown-none-softfloat.toml @@ -3,7 +3,6 @@ features = [ "alloc", "paging", "multitask", - "irq", "fd", "pipe", "epoll", @@ -12,4 +11,3 @@ features = [ ] log = "Info" max_cpu_num = 4 - diff --git a/test-suit/arceos/c/build-riscv64gc-unknown-none-elf.toml b/test-suit/arceos/c/build-riscv64gc-unknown-none-elf.toml index 384d3c3fe7..d5a7c70c0d 100644 --- a/test-suit/arceos/c/build-riscv64gc-unknown-none-elf.toml +++ b/test-suit/arceos/c/build-riscv64gc-unknown-none-elf.toml @@ -3,7 +3,6 @@ features = [ "alloc", "paging", "multitask", - "irq", "fd", "pipe", "epoll", diff --git a/test-suit/arceos/c/build-x86_64-unknown-none.toml b/test-suit/arceos/c/build-x86_64-unknown-none.toml index 384d3c3fe7..d5a7c70c0d 100644 --- a/test-suit/arceos/c/build-x86_64-unknown-none.toml +++ b/test-suit/arceos/c/build-x86_64-unknown-none.toml @@ -3,7 +3,6 @@ features = [ "alloc", "paging", "multitask", - "irq", "fd", "pipe", "epoll", diff --git a/test-suit/arceos/rust/Cargo.toml b/test-suit/arceos/rust/Cargo.toml index 72fb5e2220..4bc851d676 100644 --- a/test-suit/arceos/rust/Cargo.toml +++ b/test-suit/arceos/rust/Cargo.toml @@ -13,15 +13,15 @@ task-yield = ["ax-std", "ax-std/multitask"] sched-rr = ["ax-std", "ax-std/alloc", "ax-std/multitask", "ax-std/sched-rr"] sched-cfs = ["ax-std", "ax-std/alloc", "ax-std/multitask", "ax-std/sched-cfs"] task-affinity = ["ax-std", "ax-std/multitask"] -task-wait-queue = ["ax-std", "ax-std/multitask", "ax-std/irq"] -task-wait-queue-remote-wake = ["ax-std", "ax-std/multitask", "ax-std/irq", "ax-std/ipi"] -task-ipi = ["ax-std", "ax-std/alloc", "ax-std/multitask", "ax-std/irq", "ax-std/ipi"] -task-irq = ["ax-std", "ax-std/multitask", "ax-std/irq"] +task-wait-queue = ["ax-std", "ax-std/multitask"] +task-wait-queue-remote-wake = ["ax-std", "ax-std/multitask", "ax-std/ipi"] +task-ipi = ["ax-std", "ax-std/alloc", "ax-std/multitask", "ax-std/ipi"] +task-irq = ["ax-std", "ax-std/multitask"] task-tls = ["ax-std", "ax-std/tls", "ax-std/alloc", "ax-std/multitask"] -task-parallel = ["ax-std", "dep:rand", "ax-std/alloc", "ax-std/multitask", "ax-std/irq"] +task-parallel = ["ax-std", "dep:rand", "ax-std/alloc", "ax-std/multitask"] task-priority = ["ax-std", "ax-std/alloc", "ax-std/multitask"] -task-sleep = ["ax-std", "ax-std/multitask", "ax-std/irq"] -task-smp-online = ["ax-std", "ax-std/multitask", "ax-std/irq", "ax-std/ipi"] +task-sleep = ["ax-std", "ax-std/multitask"] +task-smp-online = ["ax-std", "ax-std/multitask", "ax-std/ipi"] task-stack-guard-page = ["ax-std", "ax-std/multitask", "ax-std/paging", "ax-std/stack-guard-page"] net-loopback = ["ax-std", "ax-driver/virtio-net", "ax-std/net", "ax-std/dns"] fs-basic = ["ax-std", "ax-driver/virtio-blk", "ax-std/alloc", "ax-std/fatfs"] diff --git a/test-suit/axvisor/normal/qemu/smoke/qemu-x86_64-vmx.toml b/test-suit/axvisor/normal/qemu/smoke/qemu-x86_64-vmx.toml index e5553095f3..1a75f2a6af 100644 --- a/test-suit/axvisor/normal/qemu/smoke/qemu-x86_64-vmx.toml +++ b/test-suit/axvisor/normal/qemu/smoke/qemu-x86_64-vmx.toml @@ -29,8 +29,6 @@ fail_regex = [ "(?i)login incorrect", "(?i)permission denied", ] -success_regex = ["(?m)^guest linux test pass!\\s*$"] -shell_prefix = "~ #" -shell_init_cmd = "pwd && echo 'guest linux test pass!'" +success_regex = ["~ #"] to_bin = false uefi = false diff --git a/test-suit/starryos/qemu-smp4/system/test-cargo-jobserver-wait/src/main.c b/test-suit/starryos/qemu-smp4/system/test-cargo-jobserver-wait/src/main.c index 6baab4cd33..db384e06e0 100644 --- a/test-suit/starryos/qemu-smp4/system/test-cargo-jobserver-wait/src/main.c +++ b/test-suit/starryos/qemu-smp4/system/test-cargo-jobserver-wait/src/main.c @@ -1071,7 +1071,18 @@ static void test_build_script_wave(void) int reaped = 0; int loops = 0; - while (reaped < BUILD_SCRIPT_WAVE && loops++ < MAX_LOOPS) { + while (loops++ < MAX_LOOPS) { + int pending_output = 0; + for (int i = 0; i < BUILD_SCRIPT_WAVE; i++) { + if (!children[i].out_eof || !children[i].err_eof) { + pending_output = 1; + break; + } + } + if (reaped >= BUILD_SCRIPT_WAVE && !pending_output) { + break; + } + struct pollfd pfds[BUILD_SCRIPT_WAVE * 2]; int child_index[BUILD_SCRIPT_WAVE * 2]; int is_stderr[BUILD_SCRIPT_WAVE * 2]; diff --git a/virtualization/axvm/Cargo.toml b/virtualization/axvm/Cargo.toml index 9f221acd3b..65bad5fe00 100644 --- a/virtualization/axvm/Cargo.toml +++ b/virtualization/axvm/Cargo.toml @@ -40,7 +40,6 @@ ax-percpu = { workspace = true, features = ["arm-el2"] } page-table-generic = { workspace = true } ax-std = { workspace = true, features = [ "paging", - "irq", "multitask", "task-ext", "smp", diff --git a/virtualization/riscv_vcpu/src/vcpu.rs b/virtualization/riscv_vcpu/src/vcpu.rs index 144c65d579..e6c9cd20e4 100644 --- a/virtualization/riscv_vcpu/src/vcpu.rs +++ b/virtualization/riscv_vcpu/src/vcpu.rs @@ -38,8 +38,13 @@ use rustsbi::{Forward, RustSBI}; use sbi_spec::{hsm, legacy, pmu, rfnc, srst}; use crate::{ - EID_HVC, RISCVVCpuCreateConfig, consts::traps::irq::S_EXT, guest_mem, regs::*, sbi_console::*, - trap::Exception, vpmu::VirtualPmu, + EID_HVC, RISCVVCpuCreateConfig, + consts::traps::irq::{S_EXT, S_SOFT}, + guest_mem, + regs::*, + sbi_console::*, + trap::Exception, + vpmu::VirtualPmu, }; unsafe extern "C" { @@ -681,6 +686,14 @@ impl RISCVVCpu { Ok(VmExit::Nothing) } + Trap::Interrupt(Interrupt::SupervisorSoft) => { + // This is a host software interrupt, commonly used for IPI and + // scheduler wakeups. Route it through the normal host IRQ path + // instead of treating it as a guest trap. + Ok(VmExit::ExternalInterrupt { + vector: S_SOFT as _, + }) + } Trap::Interrupt(Interrupt::SupervisorExternal) => { // 9 == Interrupt::SupervisorExternal //