feat(mm): page reclaim for file-backed memory pressure

os/StarryOS/kernel/src/entry.rs

@@ -22,6 +22,8 @@ pub fn init(args: &[String], envs: &[String]) {
     pseudofs::mount_all().expect("Failed to mount pseudofs");
     spawn_alarm_task();
 
+    ax_alloc::register_page_reclaim_fn(ax_fs::page_cache_reclaim);
+
     let loc = FS_CONTEXT
         .lock()
         .resolve(&args[0])

os/arceos/modules/axalloc/src/buddy_slab.rs

@@ -191,15 +191,24 @@ impl GlobalAllocator {
         alignment: usize,
         kind: UsageKind,
     ) -> AllocResult<usize> {
-        let result = self
-            .inner
-            .lock()
-            .alloc_pages(num_pages, alignment)
-            .map_err(crate::AllocError::from);
-        if result.is_ok() {
-            self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
+        let mut result = self.inner.lock().alloc_pages(num_pages, alignment);
+        if result.is_err() {
+            for _ in 0..4 {
+                let reclaimed = crate::try_page_reclaim(num_pages.max(16));
+                // Retry allocation regardless of whether reclaim ran;
+                // concurrent reclaim may have freed pages.
+                result = self.inner.lock().alloc_pages(num_pages, alignment);
+                if result.is_ok() {
+                    break;
+                }
+                if reclaimed == 0 {
+                    break;
+                }
+            }
         }
-        result
+        let addr = result.map_err(crate::AllocError::from)?;
+        self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
+        Ok(addr)
     }
 
     /// Allocates contiguous low-memory pages (physical address < 4 GiB).
@@ -209,15 +218,22 @@ impl GlobalAllocator {
         alignment: usize,
         kind: UsageKind,
     ) -> AllocResult<usize> {
-        let result = self
-            .inner
-            .lock()
-            .alloc_pages_lowmem(num_pages, alignment)
-            .map_err(crate::AllocError::from);
-        if result.is_ok() {
-            self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
+        let mut result = self.inner.lock().alloc_pages_lowmem(num_pages, alignment);
+        if result.is_err() {
+            for _ in 0..4 {
+                let reclaimed = crate::try_page_reclaim(num_pages.max(16));
+                result = self.inner.lock().alloc_pages_lowmem(num_pages, alignment);
+                if result.is_ok() {
+                    break;
+                }
+                if reclaimed == 0 {
+                    break;
+                }
+            }
         }
-        result
+        let addr = result.map_err(crate::AllocError::from)?;
+        self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
+        Ok(addr)
     }
 
     /// Allocates contiguous pages starting from the given address.

os/arceos/modules/axalloc/src/default_impl.rs

@@ -181,11 +181,22 @@ impl GlobalAllocator {
         alignment: usize,
         kind: UsageKind,
     ) -> AllocResult<usize> {
-        let addr = self
-            .palloc
-            .lock()
-            .alloc_pages(num_pages, alignment)
-            .map_err(crate::AllocError::from)?;
+        let mut result = self.palloc.lock().alloc_pages(num_pages, alignment);
+        if result.is_err() {
+            for _ in 0..4 {
+                let reclaimed = crate::try_page_reclaim(num_pages.max(16));
+                // Retry allocation regardless of whether reclaim ran;
+                // concurrent reclaim may have freed pages.
+                result = self.palloc.lock().alloc_pages(num_pages, alignment);
+                if result.is_ok() {
+                    break;
+                }
+                if reclaimed == 0 {
+                    break;
+                }
+            }
+        }
+        let addr = result.map_err(crate::AllocError::from)?;
         if !matches!(kind, UsageKind::RustHeap) {
             self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
         }

os/arceos/modules/axalloc/src/lib.rs

@@ -18,6 +18,33 @@ use strum::{IntoStaticStr, VariantArray};
 
 const PAGE_SIZE: usize = 0x1000;
 
+/// A function that tries to reclaim physical pages (e.g. by evicting
+/// clean file-backed page cache pages). Returns the number of pages freed.
+pub type PageReclaimFn = fn(num_pages: usize) -> usize;
+
+static PAGE_RECLAIM_FN: ax_kspin::SpinNoIrq<Option<PageReclaimFn>> = ax_kspin::SpinNoIrq::new(None);
+
+/// Register a callback that the allocator will invoke when a page allocation
+/// cannot be satisfied.
+pub fn register_page_reclaim_fn(f: PageReclaimFn) {
+    *PAGE_RECLAIM_FN.lock() = Some(f);
+}
+
+/// Try to reclaim physical pages by invoking the registered callback.
+/// Returns the number of pages actually freed.
+///
+/// The `SpinNoIrq` guard is released before calling into the reclaim
+/// function so that the reclaim path (and any evict listeners it
+/// triggers) runs with interrupts enabled.  Listeners registered by the
+/// address-space backend acquire blocking locks that call `might_sleep()`,
+/// which panics if interrupts are disabled.
+pub fn try_page_reclaim(num_pages: usize) -> usize {
+    // Copy out the function pointer under the lock, then release the
+    // SpinNoIrq guard before invoking the callback.
+    let reclaim_fn = { *PAGE_RECLAIM_FN.lock() };
+    reclaim_fn.map_or(0, |f| f(num_pages))
+}
+
 mod page;
 pub use page::GlobalPage;
 

os/arceos/modules/axfs-ng/src/highlevel/file.rs

@@ -6,7 +6,7 @@ use alloc::{
 use core::{
     num::NonZeroUsize,
     ops::Range,
-    sync::atomic::{AtomicU8, Ordering},
+    sync::atomic::{AtomicBool, AtomicU8, Ordering},
     task::Context,
 };
 
@@ -392,6 +392,105 @@ impl CachedFileShared {
             evict_listeners: Mutex::new(LinkedList::default()),
         }
     }
+
+    /// Scan the LRU and evict up to `max` clean pages.
+    ///
+    /// Collects non-dirty pages from the LRU cache (least-recently-used
+    /// first), pops them, notifies all registered evict listeners, and
+    /// drops the pages (freeing their backing physical memory).
+    /// Returns the number of pages evicted.
+    ///
+    /// # Lock ordering: `page_cache` → `evict_listeners`
+    ///
+    /// This function holds the `page_cache` lock while acquiring
+    /// `evict_listeners`.  Listeners must never acquire `page_cache`
+    /// (or any lock taken before `evict_listeners`), or a deadlock will
+    /// occur.  The current callers (address-space backend page-table
+    /// invalidation) do not touch the file's page cache, so this
+    /// ordering is safe in practice.
+    ///
+    /// # Context
+    ///
+    /// When called from the page-reclaim path, the reclaim callback is
+    /// invoked with interrupts enabled: `try_page_reclaim` releases its
+    /// `SpinNoIrq` guard before invoking the callback, so `might_sleep()`
+    /// in the blocking `evict_listeners.lock()` is safe.
+    fn try_evict_clean_pages(&self, max: usize) -> usize {
+        let Some(mut cache) = self.page_cache.try_lock() else {
+            return 0;
+        };
+        let mut to_evict = [0u32; 256];
+        let limit = max.min(256);
+        let mut cnt = 0;
+        for (&pn, page) in cache.iter().rev() {
+            if !page.dirty && cnt < limit {
+                to_evict[cnt] = pn;
+                cnt += 1;
+            }
+        }
+        for &pn in to_evict[..cnt].iter() {
+            if let Some(page) = cache.pop(&pn) {
+                for listener in self.evict_listeners.lock().iter() {
+                    (listener.listener)(pn, &page);
+                }
+            }
+        }
+        cnt
+    }
+}
+
+struct ReclaimGuard;
+
+impl Drop for ReclaimGuard {
+    fn drop(&mut self) {
+        RECLAIM_IN_PROGRESS.store(false, Ordering::Release);
+    }
+}
+
+static GLOBAL_CACHED_FILES: spin::RwLock<alloc::vec::Vec<Weak<CachedFileShared>>> =
+    spin::RwLock::new(alloc::vec::Vec::new());
+
+static RECLAIM_IN_PROGRESS: AtomicBool = AtomicBool::new(false);
+
+pub fn page_cache_reclaim(num_pages: usize) -> usize {
+    if RECLAIM_IN_PROGRESS.swap(true, Ordering::AcqRel) {
+        return 0;
+    }
+    let _guard = ReclaimGuard;
+
+    let mut reclaimed = 0;
+    let target = num_pages.max(16) * 2;
+    let mut file_count = 0;
+
+    if let Some(guard) = GLOBAL_CACHED_FILES.try_read() {
+        for weak in guard.iter() {
+            if let Some(file) = weak.upgrade() {
+                let freed = file.try_evict_clean_pages(target - reclaimed);
+                reclaimed += freed;
+                file_count += 1;
+                if reclaimed >= target {
+                    break;
+                }
+            }
+        }
+    } else {
+        return 0;
+    }
+
+    if reclaimed > 0 {
+        debug!(
+            "page_cache_reclaim: evicted {} clean pages across {} files",
+            reclaimed, file_count
+        );
+    }
+
+    reclaimed
+}
+
+fn register_cached_file(file: &Arc<CachedFileShared>) {
+    let mut guard = GLOBAL_CACHED_FILES.write();
+    guard.retain(|w| w.upgrade().is_some());
+    guard.push(Arc::downgrade(file));
 }
 
 /// A file handle with an LRU page cache for buffered I/O.
@@ -435,22 +534,29 @@ impl CachedFile {
         let in_memory = location.filesystem().name() == "tmpfs";
 
         let mut guard = location.user_data();
-        let shared = if let Some(shared) = guard.get::<FileUserData>().and_then(|it| it.get()) {
-            shared
-        } else {
-            let (shared, user_data) = if in_memory {
-                let shared = Arc::new(CachedFileShared::new_unbounded());
-                (shared.clone(), FileUserData::Strong(shared))
+        let (shared, is_new) =
+            if let Some(shared) = guard.get::<FileUserData>().and_then(|it| it.get()) {
+                (shared, false)
             } else {
-                let shared = Arc::new(CachedFileShared::new());
-                let user_data = FileUserData::Weak(Arc::downgrade(&shared));
-                (shared, user_data)
+                let (shared, user_data) = if in_memory {
+                    let shared = Arc::new(CachedFileShared::new_unbounded());
+                    (shared.clone(), FileUserData::Strong(shared))
+                } else {
+                    let shared = Arc::new(CachedFileShared::new());
+                    let user_data = FileUserData::Weak(Arc::downgrade(&shared));
+                    (shared, user_data)
+                };
+                guard.insert(user_data);
+                (shared, true)
             };
-            guard.insert(user_data);
-            shared
-        };
         drop(guard);
 
+        // In-memory files (tmpfs) have no backing store, so evicting clean
+        // pages would lose data. Only register disk-backed files for reclaim.
+        if is_new && !in_memory {
+            register_cached_file(&shared);
+        }
+
         Self {
             inner: location,
             shared,

os/arceos/modules/axsync/src/mutex.rs

@@ -17,10 +17,7 @@ pub struct RawMutex {
     pub(crate) lockdep: crate::lockdep::LockdepMap,
 }
 
-#[cfg(not(feature = "lockdep"))]
 pub type LockSubclass = u32;
-#[cfg(feature = "lockdep")]
-pub type LockSubclass = crate::lockdep::LockSubclass;
 
 pub trait LockdepMutexExt<T: ?Sized> {
     fn lock_nested(&self, subclass: LockSubclass) -> MutexGuard<'_, T>;
@@ -99,7 +96,7 @@ unsafe impl lock_api::RawMutex for RawMutex {
     #[track_caller]
     fn lock(&self) {
         #[cfg(feature = "lockdep")]
-        self.lock_nested(ax_lockdep::DEFAULT_LOCK_SUBCLASS);
+        self.lock_nested(0);
 
         #[cfg(not(feature = "lockdep"))]
         self.lock_plain();
@@ -110,7 +107,7 @@ unsafe impl lock_api::RawMutex for RawMutex {
     fn try_lock(&self) -> bool {
         #[cfg(feature = "lockdep")]
         {
-            self.try_lock_nested(ax_lockdep::DEFAULT_LOCK_SUBCLASS)
+            self.try_lock_nested(0)
         }
 
         #[cfg(not(feature = "lockdep"))]
@@ -183,7 +180,7 @@ impl RawMutex {
     #[inline(always)]
     #[track_caller]
     #[cfg(feature = "lockdep")]
-    fn lock_nested(&self, subclass: crate::lockdep::LockSubclass) {
+    fn lock_nested(&self, subclass: LockSubclass) {
         might_sleep();
         let current_id = current().id().as_u64();
 
@@ -196,7 +193,8 @@ impl RawMutex {
     #[track_caller]
     #[cfg(not(feature = "lockdep"))]
     fn try_lock_plain(&self) -> bool {
-        might_sleep();
+        // try_lock is a single atomic CAS — it never blocks or sleeps,
+        // so it is safe to call from atomic context (cf. Linux mutex_trylock).
         let current_id = current().id().as_u64();
         self.try_lock_after_prepare(current_id)
     }
@@ -209,8 +207,9 @@ impl RawMutex {
     #[inline(always)]
     #[track_caller]
     #[cfg(feature = "lockdep")]
-    fn try_lock_nested(&self, subclass: crate::lockdep::LockSubclass) -> bool {
-        might_sleep();
+    fn try_lock_nested(&self, subclass: LockSubclass) -> bool {
+        // try_lock is a single atomic CAS — it never blocks or sleeps,
+        // so it is safe to call from atomic context.
         let current_id = current().id().as_u64();
 
         let lockdep = crate::lockdep::LockdepAcquire::prepare_nested(self, true, subclass);