feat(mm): add page reclaim for file-backed memory pressure (rebased)

os/StarryOS/kernel/src/entry.rs

@@ -30,6 +30,8 @@ pub fn init(args: &[String], envs: &[String]) {
     pseudofs::mount_all().expect("Failed to mount pseudofs");
     spawn_alarm_task();
 
+    ax_alloc::register_page_reclaim_fn(ax_fs::page_cache_reclaim);
+
     let loc = FS_CONTEXT
         .lock()
         .resolve(&args[0])

os/StarryOS/kernel/src/mm/aspace/backend/file.rs

@@ -50,21 +50,24 @@ impl FileBackendInner {
         let aspace = Arc::downgrade(aspace);
         let handle = self.cache.add_evict_listener({
             let this = Arc::downgrade(self);
-            move |pn, _page| {
+            move |pn, _page| -> bool {
                 let Some(this) = this.upgrade() else {
-                    return;
+                    // Backend dropped — no mappings remain, safe to free.
+                    return true;
                 };
                 let Some(aspace) = aspace.upgrade() else {
                     // The address space has been dropped, nothing to do.
-                    return;
+                    return true;
                 };
                 let Some(mut aspace) = aspace.try_lock() else {
-                    // This can happen during the populate process, when new pages
-                    // are being populated and old pages are being evicted. In this
-                    // case, we delegate the unmapping to the populate process.
-                    return;
+                    // Cannot acquire AddrSpace lock (contention with populate
+                    // or another thread).  Return false so the reclaim path
+                    // puts the page back into the cache instead of freeing it
+                    // — dropping the page here would leave a dangling PTE.
+                    return false;
                 };
                 this.on_evict(pn, &mut aspace);
+                true
             }
         });
         self.handle.store(handle, Ordering::Release);

os/arceos/modules/axalloc/src/buddy_slab.rs

@@ -165,10 +165,11 @@ impl GlobalAllocator {
     /// Allocate arbitrary number of bytes. Returns the left bound of the
     /// allocated region.
     pub fn alloc(&self, layout: Layout) -> AllocResult<NonNull<u8>> {
-        let result = {
-            let inner = self.inner.lock();
-            inner.alloc(layout).map_err(crate::AllocError::from)
-        };
+        let result = self
+            .inner
+            .lock()
+            .alloc(layout)
+            .map_err(crate::AllocError::from);
         if result.is_ok() {
             self.usages.lock().alloc(UsageKind::RustHeap, layout.size());
         }
@@ -177,10 +178,9 @@ impl GlobalAllocator {
 
     /// Gives back the allocated region to the byte allocator.
     pub fn dealloc(&self, pos: NonNull<u8>, layout: Layout) {
-        {
-            let inner = self.inner.lock();
-            unsafe { inner.dealloc(pos, layout) };
-        }
+        // Lock order: inner then usages (consistent with alloc/alloc_pages).
+        // Guards are temporary — locks are never held simultaneously.
+        unsafe { self.inner.lock().dealloc(pos, layout) };
         self.usages
             .lock()
             .dealloc(UsageKind::RustHeap, layout.size());
@@ -193,16 +193,30 @@ impl GlobalAllocator {
         alignment: usize,
         kind: UsageKind,
     ) -> AllocResult<usize> {
-        let result = {
-            let inner = self.inner.lock();
-            inner
-                .alloc_pages(num_pages, alignment)
-                .map_err(crate::AllocError::from)
-        };
-        if result.is_ok() {
-            self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
+        let mut result = self.inner.lock().alloc_pages(num_pages, alignment);
+        if result.is_err() {
+            for _ in 0..4 {
+                // Reclaim num_pages (at least 16 to build free-pool headroom).
+                // page_cache_reclaim doubles this target internally.
+                // NOTE: for very large contiguous requests, reclaimed pages
+                // may be too fragmented to satisfy the allocation even when
+                // the target is met.  Consider geometric growth across retries
+                // if this becomes a problem in practice.
+                let reclaimed = crate::try_page_reclaim(num_pages.max(16));
+                // Retry allocation regardless of whether reclaim ran;
+                // concurrent reclaim may have freed pages.
+                result = self.inner.lock().alloc_pages(num_pages, alignment);
+                if result.is_ok() {
+                    break;
+                }
+                if reclaimed == 0 {
+                    break;
+                }
+            }
         }
-        result
+        let addr = result.map_err(crate::AllocError::from)?;
+        self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
+        Ok(addr)
     }
 
     /// Allocates contiguous low-memory pages (physical address < 4 GiB).
@@ -212,16 +226,22 @@ impl GlobalAllocator {
         alignment: usize,
         kind: UsageKind,
     ) -> AllocResult<usize> {
-        let result = {
-            let inner = self.inner.lock();
-            inner
-                .alloc_pages_lowmem(num_pages, alignment)
-                .map_err(crate::AllocError::from)
-        };
-        if result.is_ok() {
-            self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
+        let mut result = self.inner.lock().alloc_pages_lowmem(num_pages, alignment);
+        if result.is_err() {
+            for _ in 0..4 {
+                let reclaimed = crate::try_page_reclaim(num_pages.max(16));
+                result = self.inner.lock().alloc_pages_lowmem(num_pages, alignment);
+                if result.is_ok() {
+                    break;
+                }
+                if reclaimed == 0 {
+                    break;
+                }
+            }
         }
-        result
+        let addr = result.map_err(crate::AllocError::from)?;
+        self.usages.lock().alloc(kind, num_pages * PAGE_SIZE);
+        Ok(addr)
     }
 
     /// Allocates contiguous pages starting from the given address.
@@ -237,10 +257,9 @@ impl GlobalAllocator {
 
     /// Gives back the allocated pages starts from `pos` to the page allocator.
     pub fn dealloc_pages(&self, pos: usize, num_pages: usize, kind: UsageKind) {
-        {
-            let inner = self.inner.lock();
-            inner.dealloc_pages(pos, num_pages);
-        }
+        // Lock order: inner then usages (consistent with alloc_pages).
+        // Guards are temporary — locks are never held simultaneously.
+        self.inner.lock().dealloc_pages(pos, num_pages);
         self.usages.lock().dealloc(kind, num_pages * PAGE_SIZE);
     }
 

os/arceos/modules/axalloc/src/lib.rs

@@ -19,6 +19,29 @@ use strum::{IntoStaticStr, VariantArray};
 
 const PAGE_SIZE: usize = 0x1000;
 
+/// A function that tries to reclaim physical pages (e.g. by evicting
+/// clean file-backed page cache pages). Returns the number of pages freed.
+pub type PageReclaimFn = fn(num_pages: usize) -> usize;
+
+static PAGE_RECLAIM_FN: ax_kspin::SpinNoIrq<Option<PageReclaimFn>> = ax_kspin::SpinNoIrq::new(None);
+
+/// Register a callback that the allocator will invoke when a page allocation
+/// cannot be satisfied.
+pub fn register_page_reclaim_fn(f: PageReclaimFn) {
+    *PAGE_RECLAIM_FN.lock() = Some(f);
+}
+
+/// Try to reclaim physical pages by invoking the registered callback.
+/// Returns the number of pages actually freed.
+///
+/// The `SpinNoIrq` guard is released before calling into the reclaim
+/// function so that the reclaim path (and any evict listeners it
+/// triggers) runs with interrupts enabled.
+pub fn try_page_reclaim(num_pages: usize) -> usize {
+    let reclaim_fn = { *PAGE_RECLAIM_FN.lock() };
+    reclaim_fn.map_or(0, |f| f(num_pages))
+}
+
 mod page;
 pub use page::GlobalPage;